From 3bb86787eb7edb08a6c9b698198267bf00f8f074 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 2 Jun 2023 01:26:59 +0200 Subject: [PATCH 001/120] initial commit --- .../nonlinear_elasticity_global_rom_eqp.hpp | 343 ++++++++++++++++++ 1 file changed, 343 insertions(+) create mode 100644 examples/prom/nonlinear_elasticity_global_rom_eqp.hpp diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp new file mode 100644 index 000000000..0694ca8e3 --- /dev/null +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -0,0 +1,343 @@ +/****************************************************************************** + * + * Copyright (c) 2013-2023, Lawrence Livermore National Security, LLC + * and other libROM project developers. See the top-level COPYRIGHT + * file for details. + * + * SPDX-License-Identifier: (Apache-2.0 OR MIT) + * + *****************************************************************************/ + +// Functions used by mixed_nonlinear_diffusion.cpp with EQP. + +#include "mfem/Utilities.hpp" + +using namespace mfem; +using namespace std; + +// Element matrix assembly, copying element loop from BilinearForm::Assemble(int skip_zeros) +// and element matrix computation from HyperelasticNLFIntegrator::AssembleElementMatrix. +void AssembleElementMatrix_HyperelasticNLFIntegrator(Coefficient *Q, + const FiniteElement &el, + ElementTransformation &Trans, + DenseMatrix &elmat) +{ + +} + +// Compute coefficients of the reduced integrator with respect to inputs Q and x +// in HyperelasticNLFIntegrator_ComputeReducedEQP. +void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, + std::vector const& rw, std::vector const& qp, + const IntegrationRule *ir, + CAROM::Matrix const& V, Vector & res) +{ + +} + +void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, + std::vector const& rw, std::vector const& qp, + const IntegrationRule *ir, Coefficient *Q, + CAROM::Matrix const& V, CAROM::Vector const& x, const int rank, Vector & res) +{ + +} + +/* TODO if time... +void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, + std::vector const& qp, const IntegrationRule *ir, + Coefficient *Q, CAROM::Vector const& x, + Vector const& coef, Vector & res) +{ + +} +*/ + +// Compute a(p) u . v at all quadrature points on the given element. Coefficient Q is a(p). +void ComputeElementRowOfG(const IntegrationRule *ir, Array const& vdofs, + Coefficient *Q, Vector const& u, Vector const& v, + FiniteElement const& fe, ElementTransformation & Trans, Vector & r) +{ + MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); + int dof = fe.GetDof(); + int spaceDim = Trans.GetSpaceDim(); + + Vector u_i(spaceDim); + Vector v_i(spaceDim); + + DenseMatrix trial_vshape(dof, spaceDim); + + for (int i = 0; i < ir->GetNPoints(); i++) + { + const IntegrationPoint &ip = ir->IntPoint(i); + + Trans.SetIntPoint(&ip); + + fe.CalcVShape(Trans, trial_vshape); + + double w = Trans.Weight(); + + u_i = 0.0; + v_i = 0.0; + + for (int j=0; j= 0) ? vdofs[j] : -1 - vdofs[j]; + const double s = (vdofs[j] >= 0) ? 1.0 : -1.0; + for (int k=0; k Eval (Trans, ip); + } + + r[i] = 0.0; + for (int k=0; knumColumns(); + const int NQ = Gt.numRows(); + const int nsnap = Gt.numColumns() / NB; + + ParBilinearForm M(fespace); + + M.AddDomainIntegrator(massInteg); + M.Assemble(); + M.Finalize(); + HypreParMatrix *Mmat = M.ParallelAssemble(); + + CAROM::Matrix Mhat(NB, NB, false); + ComputeCtAB(*Mmat, *B, *B, Mhat); + Mhat.inverse(); + + CAROM::Vector PG(NB, false); + + for (int i = (snapshot >= 0 ? snapshot : 0); + i < (snapshot >= 0 ? snapshot+1 : nsnap); ++i) + { + for (int m=0; mGetNPoints(); + const int ne = fespace_R->GetNE(); + const int NB = BR->numColumns(); + const int NQ = ne * nqe; + const int nsnap = BR_snapshots->numColumns(); + + MFEM_VERIFY(nsnap == BW_snapshots->numColumns() || + nsnap + nsets == BW_snapshots->numColumns(), ""); + MFEM_VERIFY(BR->numRows() == BR_snapshots->numRows(), ""); + MFEM_VERIFY(BR->numRows() == fespace_R->GetTrueVSize(), ""); + MFEM_VERIFY(BW_snapshots->numRows() == fespace_W->GetTrueVSize(), ""); + + const bool skipFirstW = (nsnap + nsets == BW_snapshots->numColumns()); + + // Compute G of size (NB * nsnap) x NQ, but only store its transpose Gt. + CAROM::Matrix Gt(NQ, NB * nsnap, true); + + // For 0 <= j < NB, 0 <= i < nsnap, 0 <= e < ne, 0 <= m < nqe, + // G(j + (i*NB), (e*nqe) + m) + // is the coefficient of v_j^T M(p_i) V v_i at point m of element e, + // with respect to the integration rule weight at that point, + // where the "exact" quadrature solution is ir0->GetWeights(). + + Vector p_i(BW_snapshots->numRows()); + Vector v_i(BR_snapshots->numRows()); + Vector v_j(BR->numRows()); + + Vector r(nqe); + + int skip = 0; + const int nsnapPerSet = nsnap / nsets; + if (skipFirstW) + { + MFEM_VERIFY(nsets * nsnapPerSet == nsnap, ""); + skip = 1; + } + + for (int i=0; inumRows(); ++j) + p_i[j] = (*BW_snapshots)(j, i + skip); + + for (int j = 0; j < BR_snapshots->numRows(); ++j) + v_i[j] = (*BR_snapshots)(j, i); + + if (skipFirstW && i > 0 && i % nsnapPerSet == 0) + skip++; + + // Set grid function for a(p) + ParGridFunction p_gf(fespace_W); + + p_gf.SetFromTrueDofs(p_i); + + GridFunctionCoefficient p_coeff(&p_gf); + TransformedCoefficient a_coeff(&p_coeff, NonlinearCoefficient); + + ParGridFunction vi_gf(fespace_R); + vi_gf.SetFromTrueDofs(v_i); + + for (int j=0; jnumRows(); ++k) + v_j[k] = (*BR)(k, j); + + ParGridFunction vj_gf(fespace_R); + vj_gf.SetFromTrueDofs(v_j); + + // TODO: is it better to make the element loop the outer loop? + for (int e=0; e vdofs; + DofTransformation *doftrans = fespace_R->GetElementVDofs(e, vdofs); + const FiniteElement &fe = *fespace_R->GetFE(e); + ElementTransformation *eltrans = fespace_R->GetElementTransformation(e); + + ComputeElementRowOfG(ir0, vdofs, &a_coeff, vi_gf, vj_gf, fe, *eltrans, r); + + for (int m=0; m const& w_el = ir0->GetWeights(); + MFEM_VERIFY(w_el.Size() == nqe, ""); + + CAROM::Vector w(ne * nqe, true); + + for (int i=0; i elements; + Vector elemCount(pmesh->GetNE()); + elemCount = 0.0; + + for (int i=0; i 1.0e-12) + { + const int e = i / nqe; // Element index + elements.insert(e); + elemCount[e] += 1.0; + } + } + + // Empty sets, since EQP on samples inside elements. + std::set faces, edges, vertices; + CAROM::SampleVisualization(pmesh, elements, elements, faces, edges, + vertices, "EQPvis", &elemCount); +} From 312771eaaa1d7fe7ccb919545c5f138901a331c3 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 2 Jun 2023 20:27:36 +0200 Subject: [PATCH 002/120] Updated ROM operator class definition and input arguments --- .../prom/nonlinear_elasticity_global_rom.cpp | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 984f1624d..8682e9d82 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -191,6 +191,20 @@ class RomOperator : public TimeDependentOperator CAROM::Vector* z_v_librom; CAROM::Vector* z_x_librom; + // Data for EQP + bool eqp; + const IntegrationRule *ir_eqp; + std::vector eqp_rw; + std::vector eqp_qp; + Vector eqp_coef; + // const bool fastIntegration = true; //TODO: implement fast integration + + CAROM::Matrix eqp_lifting; + std::vector eqp_liftDOFs; + mutable CAROM::Vector eqp_lifted; + + int rank; + protected: CAROM::Matrix* S_hat; CAROM::Vector* S_hat_v0; @@ -215,7 +229,8 @@ class RomOperator : public TimeDependentOperator const Vector* x0_,const Vector v0_fom_,const CAROM::Matrix* V_v_, const CAROM::Matrix* V_x_, const CAROM::Matrix* U_H_, const CAROM::Matrix* Hsinv_,const int myid, const bool oversampling_, - const bool hyperreduce_,const bool x_base_only_); + const bool hyperreduce_,const bool x_base_only_, const bool use_eqp, CAROM::Vector *eqpSol, + const IntegrationRule *ir_eqp_); virtual void Mult(const Vector& y, Vector& dy_dt) const; void Mult_Hyperreduced(const Vector& y, Vector& dy_dt) const; @@ -347,6 +362,32 @@ void MergeBasis(const int dimFOM, const int nparam, const int max_num_snapshots, "mergedSV_" + name + ".txt"); } + +const CAROM::Matrix* GetSnapshotMatrix(const int dimFOM, const int nparam, + const int max_num_snapshots, std::string name) +{ + MFEM_VERIFY(nparam > 0, "Must specify a positive number of parameter sets"); + + bool update_right_SV = false; + bool isIncremental = false; + + CAROM::Options options(dimFOM, nparam * max_num_snapshots, 1, update_right_SV); + CAROM::BasisGenerator generator(options, isIncremental, "basis" + name); + + for (int paramID=0; paramID Date: Fri, 2 Jun 2023 20:34:54 +0200 Subject: [PATCH 003/120] EQP hyperreduction setup --- .../prom/nonlinear_elasticity_global_rom.cpp | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 8682e9d82..dd885400a 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -898,6 +898,26 @@ int main(int argc, char* argv[]) if (myid == 0) printf("reduced H dim = %d\n", hdim); + // Setup hyperreduction, using either EQP or sampled DOFs and a sample mesh. + + /* TODO:Change after check + CAROM::BasisReader *readerS = NULL; + ParFiniteElementSpace *sp_R_space, *sp_W_space; + CAROM::Matrix *Bsinv = NULL; + CAROM::Matrix *Ssinv = NULL; + */ + const IntegrationRule *ir0 = NULL; + + if (ir0 == NULL) + { + // int order = 2 * el.GetOrder(); + const FiniteElement &fe = *fespace.GetFE(0); + ElementTransformation *eltrans = fespace.GetElementTransformation(0); + + int order = eltrans->OrderW() + 2 * fe.GetOrder(); + ir0 = &IntRules.Get(fe.GetGeomType(), order); + } + vector num_sample_dofs_per_proc(num_procs); if (num_samples_req != -1) From 4f1bacbfb4aefa5726ad8c8351afe0e411cc0230 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 2 Jun 2023 20:53:10 +0200 Subject: [PATCH 004/120] Added hyperreduction until line 1000 --- .../prom/nonlinear_elasticity_global_rom.cpp | 72 ++++++++++--------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index dd885400a..88ca92845 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -900,12 +900,8 @@ int main(int argc, char* argv[]) // Setup hyperreduction, using either EQP or sampled DOFs and a sample mesh. - /* TODO:Change after check - CAROM::BasisReader *readerS = NULL; - ParFiniteElementSpace *sp_R_space, *sp_W_space; - CAROM::Matrix *Bsinv = NULL; - CAROM::Matrix *Ssinv = NULL; - */ + ParFiniteElementSpace* sp_XV_space; + CAROM::Matrix* Hsinv = NULL; const IntegrationRule *ir0 = NULL; if (ir0 == NULL) @@ -918,16 +914,27 @@ int main(int argc, char* argv[]) ir0 = &IntRules.Get(fe.GetGeomType(), order); } - vector num_sample_dofs_per_proc(num_procs); - - if (num_samples_req != -1) - { - nsamp_H = num_samples_req; - } - else + if (use_eqp) { - nsamp_H = hdim; - } + // EQP setup + eqpSol = new CAROM::Vector(ir0->GetNPoints() * fespace.GetNE(), true); + SetupEQP_snapshots(ir0, myid, &fespace, &W_space, nsets, BR_librom, + GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "H"), + preconditionNNLS, tolNNLS, maxNNLSnnz, *eqpSol); + + if (writeSampleMesh) WriteMeshEQP(pmesh, myid, ir0->GetNPoints(), *eqpSol); + } + else { + vector num_sample_dofs_per_proc(num_procs); + + if (num_samples_req != -1) + { + nsamp_H = num_samples_req; + } + else + { + nsamp_H = hdim; + } CAROM::Matrix* Hsinv = new CAROM::Matrix(nsamp_H, hdim, false); vector sample_dofs(nsamp_H); @@ -943,27 +950,26 @@ int main(int argc, char* argv[]) num_procs, nsamp_H); - // Construct sample mesh - const int nspaces = 1; - std::vector spfespace(nspaces); - spfespace[0] = &fespace; + // Construct sample mesh + const int nspaces = 1; + std::vector spfespace(nspaces); + spfespace[0] = &fespace; - ParFiniteElementSpace* sp_XV_space; + smm = new CAROM::SampleMeshManager(spfespace); - smm = new CAROM::SampleMeshManager(spfespace); + vector sample_dofs_empty; + vector num_sample_dofs_per_proc_empty; + num_sample_dofs_per_proc_empty.assign(num_procs, 0); - vector sample_dofs_empty; - vector num_sample_dofs_per_proc_empty; - num_sample_dofs_per_proc_empty.assign(num_procs, 0); + smm->RegisterSampledVariable("V", 0, sample_dofs, + num_sample_dofs_per_proc); + smm->RegisterSampledVariable("X", 0, sample_dofs, + num_sample_dofs_per_proc); + smm->RegisterSampledVariable("H", 0, sample_dofs, + num_sample_dofs_per_proc); - smm->RegisterSampledVariable("V", 0, sample_dofs, - num_sample_dofs_per_proc); - smm->RegisterSampledVariable("X", 0, sample_dofs, - num_sample_dofs_per_proc); - smm->RegisterSampledVariable("H", 0, sample_dofs, - num_sample_dofs_per_proc); - - smm->ConstructSampleMesh(); + smm->ConstructSampleMesh(); + } w = new CAROM::Vector(rxdim + rvdim, false); w_v = new CAROM::Vector(rvdim, false); @@ -980,7 +986,7 @@ int main(int argc, char* argv[]) int sp_size = 0; - if (myid == 0) + if (myid == 0 && !use_eqp) { sp_XV_space = smm->GetSampleFESpace(0); From 3309b340c08404067e7d34a6cd2c2f8ef9d7e121 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 2 Jun 2023 20:57:01 +0200 Subject: [PATCH 005/120] Setting BCs --- .../prom/nonlinear_elasticity_global_rom.cpp | 63 ++++++++++--------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 88ca92845..afd0e1f5d 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1040,52 +1040,55 @@ int main(int argc, char* argv[]) w_x0 = new Vector(sp_x_gf.GetTrueVector()); } - // Convert essential boundary list from FOM mesh to sample mesh - // Create binary list where 1 means essential boundary element, 0 means nonessential. - CAROM::Matrix Ess_mat(true_size, 1, true); - for (size_t i = 0; i < true_size; i++) + if (!use_eqp) { - Ess_mat(i,0) = 0; - for (size_t j = 0; j < ess_tdof_list.Size(); j++) + // Convert essential boundary list from FOM mesh to sample mesh + // Create binary list where 1 means essential boundary element, 0 means nonessential. + CAROM::Matrix Ess_mat(true_size, 1, true); + for (size_t i = 0; i < true_size; i++) { - if (ess_tdof_list[j] == i ) + Ess_mat(i,0) = 0; + for (size_t j = 0; j < ess_tdof_list.Size(); j++) { - Ess_mat(i,0) = 1; + if (ess_tdof_list[j] == i ) + { + Ess_mat(i,0) = 1; + } } } - } - // Project binary FOM list onto sampling space - MPI_Bcast(&sp_size, 1, MPI_INT, 0, MPI_COMM_WORLD); - CAROM::Matrix Ess_mat_sp(sp_size, 1, false); - smm->GatherDistributedMatrixRows("X", Ess_mat, 1, Ess_mat_sp); + // Project binary FOM list onto sampling space + MPI_Bcast(&sp_size, 1, MPI_INT, 0, MPI_COMM_WORLD); + CAROM::Matrix Ess_mat_sp(sp_size, 1, false); + smm->GatherDistributedMatrixRows("X", Ess_mat, 1, Ess_mat_sp); - // Count number of true elements in new matrix - int num_ess_sp = 0; + // Count number of true elements in new matrix + int num_ess_sp = 0; - for (size_t i = 0; i < sp_size; i++) - { - if (Ess_mat_sp(i,0) == 1) + for (size_t i = 0; i < sp_size; i++) { - num_ess_sp += 1; + if (Ess_mat_sp(i,0) == 1) + { + num_ess_sp += 1; + } } - } - // Initialize essential dof list in sampling space - Array ess_tdof_list_sp(num_ess_sp); + // Initialize essential dof list in sampling space + Array ess_tdof_list_sp(num_ess_sp); - // Add indices to list - int ctr = 0; - for (size_t i = 0; i < sp_size; i++) - { - if (Ess_mat_sp(i,0) == 1) + // Add indices to list + int ctr = 0; + for (size_t i = 0; i < sp_size; i++) { - ess_tdof_list_sp[ctr] = i; - ctr += 1; + if (Ess_mat_sp(i,0) == 1) + { + ess_tdof_list_sp[ctr] = i; + ctr += 1; + } } } - if (myid == 0) + if (myid == 0 && !use_eqp) { // Define operator in sample space soper = new HyperelasticOperator(*sp_XV_space, ess_tdof_list_sp, visc, mu, K); From 3847f42327507124b6d312f3e15a6f2424cb3e21 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 2 Jun 2023 21:36:30 +0200 Subject: [PATCH 006/120] Finished implementation in main --- examples/prom/nonlinear_elasticity_global_rom.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index afd0e1f5d..4d68d5ab0 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1094,10 +1094,11 @@ int main(int argc, char* argv[]) soper = new HyperelasticOperator(*sp_XV_space, ess_tdof_list_sp, visc, mu, K); } - if (hyperreduce) - { romop = new RomOperator(&oper, soper, rvdim, rxdim, hdim, smm, w_v0, w_x0, + if (hyperreduce) // TODO: ask about whether this is needed. + { + romop = new RomOperator(&oper, soper, rvdim, rxdim, hdim, smm, w_v0, w_x0, vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, myid, - num_samples_req != -1, hyperreduce, x_base_only); + num_samples_req != -1, hyperreduce, x_base_only, use_eqp, eqpSol, ir0); } else { @@ -1105,7 +1106,7 @@ int main(int argc, char* argv[]) &(vx0.GetBlock(0)), &(vx0.GetBlock(1)), vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, myid, - num_samples_req != -1, hyperreduce, x_base_only); + num_samples_req != -1, hyperreduce, x_base_only, use_eqp, eqpSol, ir0); } // Print lifted initial energies @@ -1160,7 +1161,7 @@ int main(int argc, char* argv[]) if (online) { - if (myid == 0) + if (myid == 0|| use_eqp) { solveTimer.Start(); ode_solver->Step(*wMFEM, t, dt_real); From cb3a526dc5a1d585f28bb73aa693cbce92ffeabb Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 2 Jun 2023 22:28:14 +0200 Subject: [PATCH 007/120] Started ROM operator initialization Need to review before finishing --- .../prom/nonlinear_elasticity_global_rom.cpp | 476 ++++++++++-------- 1 file changed, 269 insertions(+), 207 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 4d68d5ab0..f0e329dde 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -126,16 +126,16 @@ class HyperelasticOperator : public TimeDependentOperator HypreSmoother M_prec; // Preconditioner for the mass matrix M public: - HyperelasticOperator(ParFiniteElementSpace& f, Array& ess_tdof_list_, + HyperelasticOperator(ParFiniteElementSpace &f, Array &ess_tdof_list_, double visc, double mu, double K); /// Compute the right-hand side of the ODE system. - virtual void Mult(const Vector& vx, Vector& dvx_dt) const; + virtual void Mult(const Vector &vx, Vector &dvx_dt) const; - double ElasticEnergy(const ParGridFunction& x) const; - double KineticEnergy(const ParGridFunction& v) const; - void GetElasticEnergyDensity(const ParGridFunction& x, - ParGridFunction& w) const; + double ElasticEnergy(const ParGridFunction &x) const; + double KineticEnergy(const ParGridFunction &v) const; + void GetElasticEnergyDensity(const ParGridFunction &x, + ParGridFunction &w) const; mutable Vector H_sp; mutable Vector dvxdt_sp; @@ -143,11 +143,11 @@ class HyperelasticOperator : public TimeDependentOperator ParFiniteElementSpace &fespace; double viscosity; Array ess_tdof_list; - ParNonlinearForm* H; - HyperelasticModel* model; - mutable Vector z; // auxiliary vector - mutable Vector z2; // auxiliary vector - HypreParMatrix* Mmat; // Mass matrix from ParallelAssemble() + ParNonlinearForm *H; + HyperelasticModel *model; + mutable Vector z; // auxiliary vector + mutable Vector z2; // auxiliary vector + HypreParMatrix *Mmat; // Mass matrix from ParallelAssemble() HypreParMatrix Smat; virtual ~HyperelasticOperator(); @@ -160,17 +160,17 @@ class RomOperator : public TimeDependentOperator int nsamp_H; double current_dt; bool oversampling; - CAROM::Matrix* V_v_sp, * V_x_sp; - CAROM::Matrix* V_v_sp_dist; - CAROM::Vector* psp_librom, * psp_v_librom; - Vector* psp; - Vector* psp_x; - Vector* psp_v; + CAROM::Matrix *V_v_sp, *V_x_sp; + CAROM::Matrix *V_v_sp_dist; + CAROM::Vector *psp_librom, *psp_v_librom; + Vector *psp; + Vector *psp_x; + Vector *psp_v; mutable Vector zH; mutable CAROM::Vector zX; mutable CAROM::Vector zN; - const CAROM::Matrix* Hsinv; - mutable CAROM::Vector* z_librom; + const CAROM::Matrix *Hsinv; + mutable CAROM::Vector *z_librom; mutable Vector z; mutable Vector z_x; mutable Vector z_v; @@ -186,10 +186,10 @@ class RomOperator : public TimeDependentOperator mutable Vector* zfom_v; CAROM::Vector* zfom_x_librom; - CAROM::SampleMeshManager* smm; + CAROM::SampleMeshManager *smm; - CAROM::Vector* z_v_librom; - CAROM::Vector* z_x_librom; + CAROM::Vector *z_v_librom; + CAROM::Vector *z_x_librom; // Data for EQP bool eqp; @@ -206,39 +206,39 @@ class RomOperator : public TimeDependentOperator int rank; protected: - CAROM::Matrix* S_hat; - CAROM::Vector* S_hat_v0; - Vector* S_hat_v0_temp; - CAROM::Vector* S_hat_v0_temp_librom; - CAROM::Matrix* M_hat; - CAROM::Matrix* M_hat_inv; + CAROM::Matrix *S_hat; + CAROM::Vector *S_hat_v0; + Vector *S_hat_v0_temp; + CAROM::Vector *S_hat_v0_temp_librom; + CAROM::Matrix *M_hat; + CAROM::Matrix *M_hat_inv; - const CAROM::Matrix* U_H; + const CAROM::Matrix *U_H; - HyperelasticOperator* fomSp; + HyperelasticOperator *fomSp; CGSolver M_hat_solver; // Krylov solver for inverting the reduced mass matrix M_hat HypreSmoother M_hat_prec; // Preconditioner for the reduced mass matrix M_hat public: - HyperelasticOperator* fom; - - RomOperator(HyperelasticOperator* fom_, - HyperelasticOperator* fomSp_, const int rvdim_, const int rxdim_, - const int hdim_,CAROM::SampleMeshManager* smm_, const Vector* v0_, - const Vector* x0_,const Vector v0_fom_,const CAROM::Matrix* V_v_, - const CAROM::Matrix* V_x_, const CAROM::Matrix* U_H_, - const CAROM::Matrix* Hsinv_,const int myid, const bool oversampling_, - const bool hyperreduce_,const bool x_base_only_, const bool use_eqp, CAROM::Vector *eqpSol, + HyperelasticOperator *fom; + + RomOperator(HyperelasticOperator *fom_, + HyperelasticOperator *fomSp_, const int rvdim_, const int rxdim_, + const int hdim_, CAROM::SampleMeshManager *smm_, const Vector *v0_, + const Vector *x0_, const Vector v0_fom_, const CAROM::Matrix *V_v_, + const CAROM::Matrix *V_x_, const CAROM::Matrix *U_H_, + const CAROM::Matrix *Hsinv_, const int myid, const bool oversampling_, + const bool hyperreduce_, const bool x_base_only_, const bool use_eqp, CAROM::Vector *eqpSol, const IntegrationRule *ir_eqp_); - virtual void Mult(const Vector& y, Vector& dy_dt) const; - void Mult_Hyperreduced(const Vector& y, Vector& dy_dt) const; - void Mult_FullOrder(const Vector& y, Vector& dy_dt) const; + virtual void Mult(const Vector &y, Vector &dy_dt) const; + void Mult_Hyperreduced(const Vector &y, Vector &dy_dt) const; + void Mult_FullOrder(const Vector &y, Vector &dy_dt) const; CAROM::Matrix V_v, V_x, V_vTU_H; - const Vector* x0; - const Vector* v0; + const Vector *x0; + const Vector *v0; Vector v0_fom; virtual ~RomOperator(); @@ -249,33 +249,33 @@ class RomOperator : public TimeDependentOperator class ElasticEnergyCoefficient : public Coefficient { private: - HyperelasticModel& model; - const ParGridFunction& x; - DenseMatrix J; + HyperelasticModel &model; + const ParGridFunction &x; + DenseMatrix J; public: - ElasticEnergyCoefficient(HyperelasticModel& m, const ParGridFunction& x_) - : model(m), x(x_) { } - virtual double Eval(ElementTransformation& T, const IntegrationPoint& ip); - virtual ~ElasticEnergyCoefficient() { } + ElasticEnergyCoefficient(HyperelasticModel &m, const ParGridFunction &x_) + : model(m), x(x_) {} + virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip); + virtual ~ElasticEnergyCoefficient() {} }; -void InitialDeformationIC1(const Vector& x, Vector& y); +void InitialDeformationIC1(const Vector &x, Vector &y); -void InitialVelocityIC1(const Vector& x, Vector& v); +void InitialVelocityIC1(const Vector &x, Vector &v); -void InitialDeformationIC2(const Vector& x, Vector& y); +void InitialDeformationIC2(const Vector &x, Vector &y); -void InitialVelocityIC2(const Vector& x, Vector& v); +void InitialVelocityIC2(const Vector &x, Vector &v); -void visualize(ostream& out, ParMesh* mesh, ParGridFunction* deformed_nodes, - ParGridFunction* field, const char* field_name = NULL, +void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes, + ParGridFunction *field, const char *field_name = NULL, bool init_vis = false); // TODO: move this to the library? -CAROM::Matrix* GetFirstColumns(const int N, const CAROM::Matrix* A) +CAROM::Matrix *GetFirstColumns(const int N, const CAROM::Matrix *A) { - CAROM::Matrix* S = new CAROM::Matrix(A->numRows(), std::min(N, A->numColumns()), + CAROM::Matrix *S = new CAROM::Matrix(A->numRows(), std::min(N, A->numColumns()), A->distributed()); for (int i = 0; i < S->numRows(); ++i) { @@ -293,22 +293,24 @@ void BasisGeneratorFinalSummary(CAROM::BasisGenerator* bg, const std::string cutoffOutputPath) { const int rom_dim = bg->getSpatialBasis()->numColumns(); - const CAROM::Vector* sing_vals = bg->getSingularValues(); + const CAROM::Vector *sing_vals = bg->getSingularValues(); MFEM_VERIFY(rom_dim <= sing_vals->dim(), ""); double sum = 0.0; - for (int sv = 0; sv < sing_vals->dim(); ++sv) { + for (int sv = 0; sv < sing_vals->dim(); ++sv) + { sum += (*sing_vals)(sv); } - vector energy_fractions = { 0.9999999, 0.999999, 0.99999, 0.9999, 0.999, 0.99, 0.9 }; + vector energy_fractions = {0.9999999, 0.999999, 0.99999, 0.9999, 0.999, 0.99, 0.9}; bool reached_cutoff = false; ofstream outfile(cutoffOutputPath); double partialSum = 0.0; - for (int sv = 0; sv < sing_vals->dim(); ++sv) { + for (int sv = 0; sv < sing_vals->dim(); ++sv) + { partialSum += (*sing_vals)(sv); for (int i = energy_fractions.size() - 1; i >= 0; i--) { @@ -331,9 +333,9 @@ void BasisGeneratorFinalSummary(CAROM::BasisGenerator* bg, } } - if (!reached_cutoff) cutoff = sing_vals->dim(); - outfile << "Take first " << cutoff << " of " << sing_vals->dim() << - " basis vectors" << endl; + if (!reached_cutoff) + cutoff = sing_vals->dim(); + outfile << "Take first " << cutoff << " of " << sing_vals->dim() << " basis vectors" << endl; outfile.close(); } @@ -350,8 +352,7 @@ void MergeBasis(const int dimFOM, const int nparam, const int max_num_snapshots, for (int paramID = 0; paramID < nparam; ++paramID) { - std::string snapshot_filename = "basis" + std::to_string( - paramID) + "_" + name + "_snapshot"; + std::string snapshot_filename = "basis" + std::to_string(paramID) + "_" + name + "_snapshot"; generator.loadSamples(snapshot_filename, "snapshot"); } @@ -362,8 +363,7 @@ void MergeBasis(const int dimFOM, const int nparam, const int max_num_snapshots, "mergedSV_" + name + ".txt"); } - -const CAROM::Matrix* GetSnapshotMatrix(const int dimFOM, const int nparam, +const CAROM::Matrix *GetSnapshotMatrix(const int dimFOM, const int nparam, const int max_num_snapshots, std::string name) { MFEM_VERIFY(nparam > 0, "Must specify a positive number of parameter sets"); @@ -374,22 +374,21 @@ const CAROM::Matrix* GetSnapshotMatrix(const int dimFOM, const int nparam, CAROM::Options options(dimFOM, nparam * max_num_snapshots, 1, update_right_SV); CAROM::BasisGenerator generator(options, isIncremental, "basis" + name); - for (int paramID=0; paramIDdim(); @@ -399,21 +398,21 @@ void BroadcastUndistributedRomVector(CAROM::Vector* v) MFEM_VERIFY(d != 0, ""); - for (int i=0; iDimension(); // 4. Define the ODE solver used for time integration. Several implicit // singly diagonal implicit Runge-Kutta (SDIRK) methods, as well as // explicit Runge-Kutta methods are available. - ODESolver* ode_solver; + ODESolver *ode_solver; switch (ode_solver_type) { // Implicit L-stable methods @@ -610,7 +608,7 @@ int main(int argc, char* argv[]) // 6. Define a parallel mesh by a partitioning of the serial mesh. Refine // this mesh further in parallel to increase the resolution. Once the // parallel mesh is defined, the serial mesh can be deleted. - ParMesh* pmesh = new ParMesh(MPI_COMM_WORLD, *mesh); + ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, *mesh); delete mesh; for (int lev = 0; lev < par_ref_levels; lev++) { @@ -683,8 +681,8 @@ int main(int argc, char* argv[]) // 8. Set the initial conditions for v_gf, x_gf and vx, and define the // boundary conditions on a beam-like mesh (see description above). - VectorFunctionCoefficient* velo = 0; - VectorFunctionCoefficient* deform = 0; + VectorFunctionCoefficient *velo = 0; + VectorFunctionCoefficient *deform = 0; if (def_ic) { @@ -725,24 +723,24 @@ int main(int argc, char* argv[]) BlockVector vx_diff = BlockVector(vx); // Reduced order solution - Vector* wMFEM = 0; - CAROM::Vector* w = 0; - CAROM::Vector* w_v = 0; - CAROM::Vector* w_x = 0; + Vector *wMFEM = 0; + CAROM::Vector *w = 0; + CAROM::Vector *w_v = 0; + CAROM::Vector *w_x = 0; // Initialize reconstructed solution - Vector * v_rec = new Vector(v_gf.GetTrueVector()); - Vector * x_rec = new Vector(x_gf.GetTrueVector()); + Vector *v_rec = new Vector(v_gf.GetTrueVector()); + Vector *x_rec = new Vector(x_gf.GetTrueVector()); - CAROM::Vector* v_rec_librom = new CAROM::Vector(v_rec->GetData(), v_rec->Size(), - true, false); - CAROM::Vector* x_rec_librom = new CAROM::Vector(x_rec->GetData(), x_rec->Size(), - true, false); + CAROM::Vector *v_rec_librom = new CAROM::Vector(v_rec->GetData(), v_rec->Size(), + true, false); + CAROM::Vector *x_rec_librom = new CAROM::Vector(x_rec->GetData(), x_rec->Size(), + true, false); // 9. Initialize the hyperelastic operator, the GLVis visualization and print // the initial energies. HyperelasticOperator oper(fespace, ess_tdof_list, visc, mu, K); - HyperelasticOperator* soper = 0; + HyperelasticOperator *soper = 0; // Fill dvdt and dxdt Vector dvxdt(true_size * 2); @@ -753,7 +751,7 @@ int main(int argc, char* argv[]) if (visualization) { char vishost[] = "localhost"; - int visport = 19916; + int visport = 19916; vis_v.open(vishost, visport); vis_v.precision(8); visualize(vis_v, pmesh, &x_gf, &v_gf, "Velocity", true); @@ -771,7 +769,7 @@ int main(int argc, char* argv[]) // Create data collection for solution output: either VisItDataCollection for // ascii data files, or SidreDataCollection for binary data files. - DataCollection* dc = NULL; + DataCollection *dc = NULL; if (visit) { if (offline) @@ -800,37 +798,38 @@ int main(int argc, char* argv[]) } // 10. Create pROM object. - CAROM::BasisGenerator* basis_generator_v = 0; - CAROM::BasisGenerator* basis_generator_x = 0; - CAROM::BasisGenerator* basis_generator_H = 0; + CAROM::BasisGenerator *basis_generator_v = 0; + CAROM::BasisGenerator *basis_generator_x = 0; + CAROM::BasisGenerator *basis_generator_H = 0; - if (offline) { + if (offline) + { CAROM::Options options(fespace.GetTrueVSize(), max_num_snapshots, 1, update_right_SV); if (x_base_only == false) { basis_generator_v = new CAROM::BasisGenerator(options, isIncremental, - basisFileName + "_V"); + basisFileName + "_V"); } basis_generator_x = new CAROM::BasisGenerator(options, isIncremental, - basisFileName + "_X"); + basisFileName + "_X"); basis_generator_H = new CAROM::BasisGenerator(options, isIncremental, - basisFileName + "_H"); + basisFileName + "_H"); } - RomOperator* romop = 0; + RomOperator *romop = 0; - const CAROM::Matrix* BV_librom = 0; - const CAROM::Matrix* BX_librom = 0; - const CAROM::Matrix* H_librom = 0; - const CAROM::Matrix* Hsinv = 0; + const CAROM::Matrix *BV_librom = 0; + const CAROM::Matrix *BX_librom = 0; + const CAROM::Matrix *H_librom = 0; + const CAROM::Matrix *Hsinv = 0; int nsamp_H = -1; - CAROM::SampleMeshManager* smm = nullptr; + CAROM::SampleMeshManager *smm = nullptr; CAROM::Vector *eqpSol = nullptr; CAROM::Vector *eqpSol_S = nullptr; @@ -840,12 +839,11 @@ int main(int argc, char* argv[]) if (online) { // Read bases - CAROM::BasisReader* readerV = 0; + CAROM::BasisReader *readerV = 0; if (x_base_only) { - readerV = new - CAROM::BasisReader("basisX"); // The basis for v uses the x basis instead. + readerV = new CAROM::BasisReader("basisX"); // The basis for v uses the x basis instead. rvdim = rxdim; } else @@ -900,8 +898,8 @@ int main(int argc, char* argv[]) // Setup hyperreduction, using either EQP or sampled DOFs and a sample mesh. - ParFiniteElementSpace* sp_XV_space; - CAROM::Matrix* Hsinv = NULL; + ParFiniteElementSpace *sp_XV_space; + CAROM::Matrix *Hsinv = NULL; const IntegrationRule *ir0 = NULL; if (ir0 == NULL) @@ -922,9 +920,11 @@ int main(int argc, char* argv[]) GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "H"), preconditionNNLS, tolNNLS, maxNNLSnnz, *eqpSol); - if (writeSampleMesh) WriteMeshEQP(pmesh, myid, ir0->GetNPoints(), *eqpSol); - } - else { + if (writeSampleMesh) + WriteMeshEQP(pmesh, myid, ir0->GetNPoints(), *eqpSol); + } + else + { vector num_sample_dofs_per_proc(num_procs); if (num_samples_req != -1) @@ -952,7 +952,7 @@ int main(int argc, char* argv[]) // Construct sample mesh const int nspaces = 1; - std::vector spfespace(nspaces); + std::vector spfespace(nspaces); spfespace[0] = &fespace; smm = new CAROM::SampleMeshManager(spfespace); @@ -962,14 +962,14 @@ int main(int argc, char* argv[]) num_sample_dofs_per_proc_empty.assign(num_procs, 0); smm->RegisterSampledVariable("V", 0, sample_dofs, - num_sample_dofs_per_proc); + num_sample_dofs_per_proc); smm->RegisterSampledVariable("X", 0, sample_dofs, - num_sample_dofs_per_proc); + num_sample_dofs_per_proc); smm->RegisterSampledVariable("H", 0, sample_dofs, - num_sample_dofs_per_proc); + num_sample_dofs_per_proc); smm->ConstructSampleMesh(); - } + } w = new CAROM::Vector(rxdim + rvdim, false); w_v = new CAROM::Vector(rvdim, false); @@ -981,8 +981,8 @@ int main(int argc, char* argv[]) wMFEM = new Vector(&((*w)(0)), rxdim + rvdim); // Initial conditions - Vector* w_v0 = 0; - Vector* w_x0 = 0; + Vector *w_v0 = 0; + Vector *w_x0 = 0; int sp_size = 0; @@ -1007,8 +1007,8 @@ int main(int argc, char* argv[]) sp_v_gf.MakeTRef(sp_XV_space, sp_vx, sp_offset[0]); sp_x_gf.MakeTRef(sp_XV_space, sp_vx, sp_offset[1]); - VectorFunctionCoefficient* velo = 0; - VectorFunctionCoefficient* deform = 0; + VectorFunctionCoefficient *velo = 0; + VectorFunctionCoefficient *deform = 0; if (def_ic) { @@ -1047,12 +1047,12 @@ int main(int argc, char* argv[]) CAROM::Matrix Ess_mat(true_size, 1, true); for (size_t i = 0; i < true_size; i++) { - Ess_mat(i,0) = 0; + Ess_mat(i, 0) = 0; for (size_t j = 0; j < ess_tdof_list.Size(); j++) { - if (ess_tdof_list[j] == i ) + if (ess_tdof_list[j] == i) { - Ess_mat(i,0) = 1; + Ess_mat(i, 0) = 1; } } } @@ -1067,7 +1067,7 @@ int main(int argc, char* argv[]) for (size_t i = 0; i < sp_size; i++) { - if (Ess_mat_sp(i,0) == 1) + if (Ess_mat_sp(i, 0) == 1) { num_ess_sp += 1; } @@ -1080,7 +1080,7 @@ int main(int argc, char* argv[]) int ctr = 0; for (size_t i = 0; i < sp_size; i++) { - if (Ess_mat_sp(i,0) == 1) + if (Ess_mat_sp(i, 0) == 1) { ess_tdof_list_sp[ctr] = i; ctr += 1; @@ -1095,7 +1095,7 @@ int main(int argc, char* argv[]) } if (hyperreduce) // TODO: ask about whether this is needed. - { + { romop = new RomOperator(&oper, soper, rvdim, rxdim, hdim, smm, w_v0, w_x0, vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, myid, num_samples_req != -1, hyperreduce, x_base_only, use_eqp, eqpSol, ir0); @@ -1112,10 +1112,10 @@ int main(int argc, char* argv[]) // Print lifted initial energies BroadcastUndistributedRomVector(w); - for (int i=0; iV_v.mult(*w_v, *v_rec_librom); @@ -1134,11 +1134,9 @@ int main(int argc, char* argv[]) { cout << "Lifted initial energies, EE = " << ee << ", KE = " << ke << ", ΔTE = " << (ee + ke) - (ee0 + ke0) << endl; - } ode_solver->Init(*romop); - } else { @@ -1161,7 +1159,7 @@ int main(int argc, char* argv[]) if (online) { - if (myid == 0|| use_eqp) + if (myid == 0 || use_eqp) { solveTimer.Start(); ode_solver->Step(*wMFEM, t, dt_real); @@ -1182,8 +1180,7 @@ int main(int argc, char* argv[]) if (offline) { - if (basis_generator_x->isNextSample(t) || x_base_only == false - && basis_generator_v->isNextSample(t)) + if (basis_generator_x->isNextSample(t) || x_base_only == false && basis_generator_v->isNextSample(t)) { dvxdt = oper.dvxdt_sp.GetData(); vx_diff = BlockVector(vx); @@ -1218,10 +1215,10 @@ int main(int argc, char* argv[]) { BroadcastUndistributedRomVector(w); - for (int i=0; iV_v.mult(*w_v, *v_rec_librom); @@ -1232,13 +1229,11 @@ int main(int argc, char* argv[]) v_gf.SetFromTrueDofs(*v_rec); x_gf.SetFromTrueDofs(*x_rec); - } else { v_gf.SetFromTrueVector(); x_gf.SetFromTrueVector(); - } double ee = oper.ElasticEnergy(x_gf); @@ -1262,7 +1257,7 @@ int main(int argc, char* argv[]) if (visit) { - GridFunction* nodes = &x_gf; + GridFunction *nodes = &x_gf; int owns_nodes = 0; pmesh->SwapNodes(nodes, owns_nodes); @@ -1274,13 +1269,13 @@ int main(int argc, char* argv[]) } // timestep loop - if (myid == 0) cout << "Elapsed time for time integration loop " << - solveTimer.RealTime() << endl; + if (myid == 0) + cout << "Elapsed time for time integration loop " << solveTimer.RealTime() << endl; ostringstream velo_name, pos_name; - velo_name << "velocity_s"<< s << "." << setfill('0') << setw(6) << myid; - pos_name << "position_s"<< s << "." << setfill('0') << setw(6) << myid; + velo_name << "velocity_s" << s << "." << setfill('0') << setw(6) << myid; + pos_name << "position_s" << s << "." << setfill('0') << setw(6) << myid; if (offline) { @@ -1307,7 +1302,7 @@ int main(int argc, char* argv[]) delete basis_generator_x; // 14. Save the displaced mesh, the velocity and elastic energy. - GridFunction* nodes = &x_gf; + GridFunction *nodes = &x_gf; int owns_nodes = 0; pmesh->SwapNodes(nodes, owns_nodes); @@ -1341,7 +1336,6 @@ int main(int argc, char* argv[]) ee_ofs.precision(8); oper.GetElasticEnergyDensity(x_gf, w_gf); w_gf.Save(ee_ofs); - } // 15. Calculate the relative error between the ROM final solution and the true solution. @@ -1375,9 +1369,9 @@ int main(int argc, char* argv[]) double tot_diff_norm_x = sqrt(InnerProduct(MPI_COMM_WORLD, diff_x, diff_x)); double tot_v_fom_norm = sqrt(InnerProduct(MPI_COMM_WORLD, - v_fom, v_fom)); + v_fom, v_fom)); double tot_x_fom_norm = sqrt(InnerProduct(MPI_COMM_WORLD, - x_fom, x_fom)); + x_fom, x_fom)); if (myid == 0) { @@ -1393,28 +1387,29 @@ int main(int argc, char* argv[]) delete pmesh; totalTimer.Stop(); - if (myid == 0) cout << "Elapsed time for entire simulation " << - totalTimer.RealTime() << endl; + if (myid == 0) + cout << "Elapsed time for entire simulation " << totalTimer.RealTime() << endl; MPI_Finalize(); return 0; } -void visualize(ostream& out, ParMesh* mesh, ParGridFunction* deformed_nodes, - ParGridFunction* field, const char* field_name, bool init_vis) +void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes, + ParGridFunction *field, const char *field_name, bool init_vis) { if (!out) { return; } - GridFunction* nodes = deformed_nodes; + GridFunction *nodes = deformed_nodes; int owns_nodes = 0; mesh->SwapNodes(nodes, owns_nodes); out << "parallel " << mesh->GetNRanks() << " " << mesh->GetMyRank() << "\n"; - out << "solution\n" << *mesh << *field; + out << "solution\n" + << *mesh << *field; mesh->SwapNodes(nodes, owns_nodes); @@ -1433,14 +1428,14 @@ void visualize(ostream& out, ParMesh* mesh, ParGridFunction* deformed_nodes, out << flush; } -HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace& f, - Array& ess_tdof_list_, double visc, - double mu, double K) +HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f, + Array &ess_tdof_list_, double visc, + double mu, double K) : TimeDependentOperator(2 * f.TrueVSize(), 0.0), fespace(f), ess_tdof_list(ess_tdof_list_), M(NULL), S(NULL), H(NULL), viscosity(visc), M_solver(f.GetComm()), - z(height / 2), z2(height / 2), H_sp(height/2), dvxdt_sp(height/2) + z(height / 2), z2(height / 2), H_sp(height / 2), dvxdt_sp(height / 2) { const double rel_tol = 1e-8; const int skip_zero_entries = 0; @@ -1453,7 +1448,7 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace& f, M->Assemble(skip_zero_entries); M->Finalize(skip_zero_entries); Mmat = M->ParallelAssemble(); - HypreParMatrix* Me = Mmat->EliminateRowsCols(ess_tdof_list); + HypreParMatrix *Me = Mmat->EliminateRowsCols(ess_tdof_list); delete Me; M_solver.iterative_mode = false; @@ -1478,7 +1473,7 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace& f, S->FormSystemMatrix(ess_tdof_list, Smat); } -void HyperelasticOperator::Mult(const Vector& vx, Vector& dvx_dt) const +void HyperelasticOperator::Mult(const Vector &vx, Vector &dvx_dt) const { // Create views to the sub-vectors v, x of vx, and dv_dt, dx_dt of dvx_dt int sc = height / 2; @@ -1503,12 +1498,12 @@ void HyperelasticOperator::Mult(const Vector& vx, Vector& dvx_dt) const dvxdt_sp = dvx_dt; } -double HyperelasticOperator::ElasticEnergy(const ParGridFunction& x) const +double HyperelasticOperator::ElasticEnergy(const ParGridFunction &x) const { return H->GetEnergy(x); } -double HyperelasticOperator::KineticEnergy(const ParGridFunction& v) const +double HyperelasticOperator::KineticEnergy(const ParGridFunction &v) const { double loc_energy = 0.5 * M->InnerProduct(v, v); double energy; @@ -1518,7 +1513,7 @@ double HyperelasticOperator::KineticEnergy(const ParGridFunction& v) const } void HyperelasticOperator::GetElasticEnergyDensity( - const ParGridFunction& x, ParGridFunction& w) const + const ParGridFunction &x, ParGridFunction &w) const { ElasticEnergyCoefficient w_coeff(*model, x); w.ProjectCoefficient(w_coeff); @@ -1533,8 +1528,8 @@ HyperelasticOperator::~HyperelasticOperator() delete H; } -double ElasticEnergyCoefficient::Eval(ElementTransformation& T, - const IntegrationPoint& ip) +double ElasticEnergyCoefficient::Eval(ElementTransformation &T, + const IntegrationPoint &ip) { model.SetTransformation(T); x.GetVectorGradient(T, J); @@ -1547,61 +1542,68 @@ void InitialDeformationIC1(const Vector &x, Vector &y) y = x; } -void InitialVelocityIC1(const Vector& x, Vector& v) +void InitialVelocityIC1(const Vector &x, Vector &v) { const int dim = x.Size(); const double s_eff = s / 80.0; v = 0.0; - v(dim - 1) = -s_eff * sin(s*x(0)); + v(dim - 1) = -s_eff * sin(s * x(0)); } -void InitialDeformationIC2(const Vector &x, Vector &y) //See MFEM ex19 +void InitialDeformationIC2(const Vector &x, Vector &y) // See MFEM ex19 { // Set the initial configuration. Having this different from the reference // configuration can help convergence const int dim = x.Size(); const double s_eff = s; y = x; - y(dim-1) = x(dim - 1) + 0.25*x(0) * s_eff; + y(dim - 1) = x(dim - 1) + 0.25 * x(0) * s_eff; } -void InitialVelocityIC2(const Vector& x, Vector& v) +void InitialVelocityIC2(const Vector &x, Vector &v) { v = 0.0; } -RomOperator::RomOperator(HyperelasticOperator* fom_, - HyperelasticOperator* fomSp_, const int rvdim_, const int rxdim_, - const int hdim_, CAROM::SampleMeshManager* smm_, const Vector* v0_, - const Vector* x0_, const Vector v0_fom_, const CAROM::Matrix* V_v_, - const CAROM::Matrix* V_x_, const CAROM::Matrix* U_H_, - const CAROM::Matrix* Hsinv_, const int myid, const bool oversampling_, - const bool hyperreduce_, const bool x_base_only_) +RomOperator::RomOperator(HyperelasticOperator *fom_, + HyperelasticOperator *fomSp_, const int rvdim_, const int rxdim_, + const int hdim_, CAROM::SampleMeshManager *smm_, const Vector *v0_, + const Vector *x0_, const Vector v0_fom_, const CAROM::Matrix *V_v_, + const CAROM::Matrix *V_x_, const CAROM::Matrix *U_H_, + const CAROM::Matrix *Hsinv_, const int myid, const bool oversampling_, + const bool hyperreduce_, const bool x_base_only_, const bool use_eqp, + CAROM::Vector *eqpSol, const IntegrationRule *ir_eqp_) : TimeDependentOperator(rxdim_ + rvdim_, 0.0), fom(fom_), fomSp(fomSp_), rxdim(rxdim_), rvdim(rvdim_), hdim(hdim_), x0(x0_), v0(v0_), v0_fom(v0_fom_), smm(smm_), nsamp_H(smm_->GetNumVarSamples("H")), V_x(*V_x_), V_v(*V_v_), U_H(U_H_), Hsinv(Hsinv_), zN(std::max(nsamp_H, 1), false), zX(std::max(nsamp_H, - 1), false), M_hat_solver(fom_->fespace.GetComm()), oversampling(oversampling_), - z(height / 2), hyperreduce(hyperreduce_), x_base_only(x_base_only_) + 1), + false), + M_hat_solver(fom_->fespace.GetComm()), oversampling(oversampling_), + z(height / 2), hyperreduce(hyperreduce_), x_base_only(x_base_only_), eqp(use_eqp), + ir_eqp(ir_eqp_) { - if (myid == 0) + if (myid == 0 && !eqp) { V_v_sp = new CAROM::Matrix(fomSp->Height() / 2, rvdim, false); V_x_sp = new CAROM::Matrix(fomSp->Height() / 2, rxdim, false); } // Gather distributed vectors - if (x_base_only) + if (!eqp) { - smm->GatherDistributedMatrixRows("X", V_v, rvdim, *V_v_sp); - } - else - { - smm->GatherDistributedMatrixRows("V", V_v, rvdim, *V_v_sp); - } + if (x_base_only) + { + smm->GatherDistributedMatrixRows("X", V_v, rvdim, *V_v_sp); + } + else + { + smm->GatherDistributedMatrixRows("V", V_v, rvdim, *V_v_sp); + } - smm->GatherDistributedMatrixRows("X", V_x, rxdim, *V_x_sp); + smm->GatherDistributedMatrixRows("X", V_x, rxdim, *V_x_sp); + } // Create V_vTU_H, for hyperreduction V_v.transposeMult(*U_H, V_vTU_H); @@ -1610,7 +1612,7 @@ RomOperator::RomOperator(HyperelasticOperator* fom_, S_hat_v0 = new CAROM::Vector(rvdim, false); S_hat_v0_temp = new Vector(v0_fom.Size()); S_hat_v0_temp_librom = new CAROM::Vector(S_hat_v0_temp->GetData(), - S_hat_v0_temp->Size(), true, false); + S_hat_v0_temp->Size(), true, false); M_hat = new CAROM::Matrix(rvdim, rvdim, false); M_hat_inv = new CAROM::Matrix(rvdim, rvdim, false); @@ -1627,9 +1629,9 @@ RomOperator::RomOperator(HyperelasticOperator* fom_, // Invert M_hat and store M_hat->inverse(*M_hat_inv); - if (myid == 0) + if (myid == 0 && !eqp) { - const int spdim = fomSp->Height(); // Reduced height + const int spdim = fomSp->Height(); // Reduced height zH.SetSize(spdim / 2); // Samples of H @@ -1652,10 +1654,9 @@ RomOperator::RomOperator(HyperelasticOperator* fom_, psp_v_librom = new CAROM::Vector(psp_v->GetData(), psp_v->Size(), false, false); } + const int fdim = fom->Height(); // Unreduced height if (!hyperreduce) { - const int fdim = fom->Height(); // Unreduced height - z.SetSize(fdim / 2); z_v.SetSize(fdim / 2); z_x.SetSize(fdim / 2); @@ -1677,7 +1678,70 @@ RomOperator::RomOperator(HyperelasticOperator* fom_, pfom_v_librom = new CAROM::Vector(pfom_v->GetData(), pfom_v->Size(), true, false); } + else if (eqp) + { + pfom_v_librom = new CAROM::Vector(fdim / 2, true); + pfom_v = new Vector(pfom_v_librom->getData(), fdim / 2); + + std::set elements; + const int nqe = ir_eqp->GetWeights().Size(); + for (int i = 0; i < eqpSol->dim(); ++i) + { + if ((*eqpSol)(i) > 1.0e-12) + { + const int e = i / nqe; // Element index + elements.insert(e); + eqp_rw.push_back((*eqpSol)(i)); + eqp_qp.push_back(i); + } + } + + cout << myid << ": EQP using " << elements.size() << " elements out of " + << fom->fespace_R.GetNE() << endl; + + // TODO: implement the one below + GetEQPCoefficients_HyperelasticNLFIntegrator(); + + // TODO: Understand this + // Set the matrix of the linear map from the ROM W space to all DOFs on + // sampled elements. + Array vdofs; + int numSampledDofs = 0; + for (auto e : elements) + { + fom_->fespace_W.GetElementVDofs(e, vdofs); + numSampledDofs += vdofs.Size(); + } + + eqp_lifting.setSize(numSampledDofs, rwdim); + eqp_liftDOFs.resize(numSampledDofs); + eqp_lifted.setSize(numSampledDofs); + + int cnt = 0; + for (auto e : elements) + { + fom_->fespace_W.GetElementVDofs(e, vdofs); + for (int i = 0; i < vdofs.Size(); ++i, ++cnt) + eqp_liftDOFs[cnt] = vdofs[i]; + } + MFEM_VERIFY(cnt == numSampledDofs, ""); + + CAROM::Vector ej(rwdim, false); + + for (int j = 0; j < rwdim; ++j) + { + ej = 0.0; + ej(j) = 1.0; + V_W.mult(ej, *pfom_W_librom); + p_gf.SetFromTrueDofs(*pfom_W); + + for (int i = 0; i < numSampledDofs; ++i) + { + eqp_lifting(i, j) = (*pfom_W)[eqp_liftDOFs[i]]; + } + } + } } RomOperator::~RomOperator() @@ -1687,7 +1751,7 @@ RomOperator::~RomOperator() delete M_hat_inv; } -void RomOperator::Mult_Hyperreduced(const Vector& vx, Vector& dvx_dt) const +void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const { // Check that the sizes match MFEM_VERIFY(vx.Size() == rvdim + rxdim && dvx_dt.Size() == rvdim + rxdim, ""); @@ -1741,10 +1805,9 @@ void RomOperator::Mult_Hyperreduced(const Vector& vx, Vector& dvx_dt) const dv_dt_librom); // to invert reduced mass matrix operator. V_x_sp->transposeMult(*psp_v_librom, dx_dt_librom); - } -void RomOperator::Mult_FullOrder(const Vector& vx, Vector& dvx_dt) const +void RomOperator::Mult_FullOrder(const Vector &vx, Vector &dvx_dt) const { // Check that the sizes match MFEM_VERIFY(vx.Size() == rvdim + rxdim && dvx_dt.Size() == rvdim + rxdim, ""); @@ -1783,10 +1846,9 @@ void RomOperator::Mult_FullOrder(const Vector& vx, Vector& dvx_dt) const dv_dt_librom); // to invert reduced mass matrix operator. V_x.transposeMult(*pfom_v_librom, dx_dt_librom); - } -void RomOperator::Mult(const Vector& vx, Vector& dvx_dt) const +void RomOperator::Mult(const Vector &vx, Vector &dvx_dt) const { if (hyperreduce) From 67efefc99f6b45516c957fdbb22aa83e243aac66 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 2 Jun 2023 22:48:44 +0200 Subject: [PATCH 008/120] Gone through the code and implemented what was straightforward --- examples/prom/nonlinear_elasticity_global_rom.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index f0e329dde..538aaf09b 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1764,8 +1764,10 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const Vector dx_dt(dvx_dt.GetData() + rvdim, rxdim); CAROM::Vector dv_dt_librom(dv_dt.GetData(), rvdim, false, false); CAROM::Vector dx_dt_librom(dx_dt.GetData(), rxdim, false, false); - - // Lift the x-, and v-vectors + if (eqp) + {} + else + {// Lift the x-, and v-vectors // I.e. perform v = v0 + V_v v^, where v^ is the input V_v_sp->mult(v_librom, *z_v_librom); V_x_sp->mult(x_librom, *z_x_librom); @@ -1791,7 +1793,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const } // Multiply by V_v^T * U_H - V_vTU_H.mult(zX, z_librom); + V_vTU_H.mult(zX, z_librom);} if (fomSp->viscosity != 0.0) { From f686e006a0ca14ca191ddd95c0cdbe8674c19494 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Sun, 4 Jun 2023 21:13:52 +0200 Subject: [PATCH 009/120] Mult_hyperreduced --- .../prom/nonlinear_elasticity_global_rom.cpp | 76 +++++++++++++------ 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 538aaf09b..a9532cf1e 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1764,36 +1764,68 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const Vector dx_dt(dvx_dt.GetData() + rvdim, rxdim); CAROM::Vector dv_dt_librom(dv_dt.GetData(), rvdim, false, false); CAROM::Vector dx_dt_librom(dx_dt.GetData(), rxdim, false, false); - if (eqp) - {} - else - {// Lift the x-, and v-vectors + // Lift v-vector // I.e. perform v = v0 + V_v v^, where v^ is the input V_v_sp->mult(v_librom, *z_v_librom); - V_x_sp->mult(x_librom, *z_x_librom); - - add(z_v, *v0, *psp_v); // Store liftings - add(z_x, *x0, *psp_x); - // Hyperreduce H - // Apply H to x to get zH - fomSp->H->Mult(*psp_x, zH); + // Store lifting + add(z_v, *v0, *psp_v); - // Sample the values from zH - smm->GetSampledValues("H", zH, zN); - - // Apply inverse H-basis - if (oversampling) + if (eqp) { - Hsinv->transposeMult(zN, zX); + Vector resEQP; + if (fastIntegration) + VectorFEMassIntegrator_ComputeReducedEQP_Fast(&(fom->fespace_R), eqp_qp, + ir_eqp, &a_coeff, + x_librom, eqp_coef, resEQP); + else + VectorFEMassIntegrator_ComputeReducedEQP(&(fom->fespace_R), eqp_rw, + eqp_qp, ir_eqp, &a_coeff, + V_x, x_librom, rank, resEQP); + + Vector recv(resEQP); + MPI_Allreduce(resEQP.GetData(), recv.GetData(), resEQP.Size(), MPI_DOUBLE, + MPI_SUM, MPI_COMM_WORLD); + resEQP = recv; + + // NOTE: in the hyperreduction case, the residual is of dimension nldim, + // which is the dimension of the ROM space for the nonlinear term. + // In the EQP case, there is no use of a ROM space for the nonlinear + // term. Instead, the FOM computation of the nonlinear term is + // approximated by the reduced quadrature rule in the FOM space. + // Therefore, the residual here is of dimension rrdim. + + MFEM_VERIFY(resEQP.Size() == rrdim, ""); + for (int i = 0; i < rrdim; ++i) + z_librom[i] += resEQP[i]; } else - { - Hsinv->mult(zN, zX); - } + { // Lift the x-vector + V_x_sp->mult(x_librom, *z_x_librom); + + // Store lifting + add(z_x, *x0, *psp_x); - // Multiply by V_v^T * U_H - V_vTU_H.mult(zX, z_librom);} + // Hyperreduce H + // Apply H to x to get zH + fomSp->H->Mult(*psp_x, zH); + + // Sample the values from zH + smm->GetSampledValues("H", zH, zN); + + // Apply inverse H-basis + if (oversampling) + { + Hsinv->transposeMult(zN, zX); + } + else + { + Hsinv->mult(zN, zX); + } + + // Multiply by V_v^T * U_H + V_vTU_H.mult(zX, z_librom); + } if (fomSp->viscosity != 0.0) { From ec129a84eb4e9bb610d8b879114249a111706b90 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Sun, 4 Jun 2023 21:29:17 +0200 Subject: [PATCH 010/120] small fixes to make consistent with class definition --- .../prom/nonlinear_elasticity_global_rom.cpp | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index a9532cf1e..d14599bb0 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1764,24 +1764,21 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const Vector dx_dt(dvx_dt.GetData() + rvdim, rxdim); CAROM::Vector dv_dt_librom(dv_dt.GetData(), rvdim, false, false); CAROM::Vector dx_dt_librom(dx_dt.GetData(), rxdim, false, false); - // Lift v-vector - // I.e. perform v = v0 + V_v v^, where v^ is the input - V_v_sp->mult(v_librom, *z_v_librom); - - // Store lifting - add(z_v, *v0, *psp_v); if (eqp) - { + { // Lift v-vector and save + V_v.mult(v_librom, *z_v_librom); + V_x->transposeMult(*z_v_librom, dx_dt_librom); + Vector resEQP; if (fastIntegration) - VectorFEMassIntegrator_ComputeReducedEQP_Fast(&(fom->fespace_R), eqp_qp, - ir_eqp, &a_coeff, - x_librom, eqp_coef, resEQP); + HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace_R), eqp_qp, + ir_eqp, &a_coeff, + x_librom, eqp_coef, resEQP); else - VectorFEMassIntegrator_ComputeReducedEQP(&(fom->fespace_R), eqp_rw, - eqp_qp, ir_eqp, &a_coeff, - V_x, x_librom, rank, resEQP); + HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace_R), eqp_rw, + eqp_qp, ir_eqp, &a_coeff, + V_x, x_librom, rank, resEQP); Vector recv(resEQP); MPI_Allreduce(resEQP.GetData(), recv.GetData(), resEQP.Size(), MPI_DOUBLE, @@ -1800,10 +1797,13 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const z_librom[i] += resEQP[i]; } else - { // Lift the x-vector + { // Lift x- and v-vector + // I.e. perform v = v0 + V_v v^, where v^ is the input + V_v_sp->mult(v_librom, *z_v_librom); V_x_sp->mult(x_librom, *z_x_librom); - // Store lifting + // Store liftings + add(z_v, *v0, *psp_v); add(z_x, *x0, *psp_x); // Hyperreduce H @@ -1825,6 +1825,9 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const // Multiply by V_v^T * U_H V_vTU_H.mult(zX, z_librom); + + // store dx_dt + V_x_sp->transposeMult(*psp_v_librom, dx_dt_librom); } if (fomSp->viscosity != 0.0) @@ -1838,7 +1841,6 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const M_hat_inv->mult(*z_librom, dv_dt_librom); // to invert reduced mass matrix operator. - V_x_sp->transposeMult(*psp_v_librom, dx_dt_librom); } void RomOperator::Mult_FullOrder(const Vector &vx, Vector &dvx_dt) const From 5bd2f90f8eaf902b47c971945cc1cad690973b86 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Sun, 4 Jun 2023 21:34:56 +0200 Subject: [PATCH 011/120] removed eqp_lifting code, since it's not necessary --- .../prom/nonlinear_elasticity_global_rom.cpp | 57 ++----------------- 1 file changed, 6 insertions(+), 51 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index d14599bb0..0ff8c9747 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -199,10 +199,6 @@ class RomOperator : public TimeDependentOperator Vector eqp_coef; // const bool fastIntegration = true; //TODO: implement fast integration - CAROM::Matrix eqp_lifting; - std::vector eqp_liftDOFs; - mutable CAROM::Vector eqp_lifted; - int rank; protected: @@ -1655,7 +1651,7 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, } const int fdim = fom->Height(); // Unreduced height - if (!hyperreduce) + if (!hyperreduce || eqp) { z.SetSize(fdim / 2); z_v.SetSize(fdim / 2); @@ -1678,11 +1674,9 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, pfom_v_librom = new CAROM::Vector(pfom_v->GetData(), pfom_v->Size(), true, false); } - else if (eqp) - { - pfom_v_librom = new CAROM::Vector(fdim / 2, true); - pfom_v = new Vector(pfom_v_librom->getData(), fdim / 2); + if (eqp) + { std::set elements; const int nqe = ir_eqp->GetWeights().Size(); @@ -1698,49 +1692,10 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, } cout << myid << ": EQP using " << elements.size() << " elements out of " - << fom->fespace_R.GetNE() << endl; + << fom->fespace.GetNE() << endl; // TODO: implement the one below GetEQPCoefficients_HyperelasticNLFIntegrator(); - - // TODO: Understand this - // Set the matrix of the linear map from the ROM W space to all DOFs on - // sampled elements. - Array vdofs; - int numSampledDofs = 0; - for (auto e : elements) - { - fom_->fespace_W.GetElementVDofs(e, vdofs); - numSampledDofs += vdofs.Size(); - } - - eqp_lifting.setSize(numSampledDofs, rwdim); - eqp_liftDOFs.resize(numSampledDofs); - eqp_lifted.setSize(numSampledDofs); - - int cnt = 0; - for (auto e : elements) - { - fom_->fespace_W.GetElementVDofs(e, vdofs); - for (int i = 0; i < vdofs.Size(); ++i, ++cnt) - eqp_liftDOFs[cnt] = vdofs[i]; - } - MFEM_VERIFY(cnt == numSampledDofs, ""); - - CAROM::Vector ej(rwdim, false); - - for (int j = 0; j < rwdim; ++j) - { - ej = 0.0; - ej(j) = 1.0; - V_W.mult(ej, *pfom_W_librom); - p_gf.SetFromTrueDofs(*pfom_W); - - for (int i = 0; i < numSampledDofs; ++i) - { - eqp_lifting(i, j) = (*pfom_W)[eqp_liftDOFs[i]]; - } - } } } @@ -1772,11 +1727,11 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const Vector resEQP; if (fastIntegration) - HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace_R), eqp_qp, + HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), eqp_qp, ir_eqp, &a_coeff, x_librom, eqp_coef, resEQP); else - HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace_R), eqp_rw, + HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, &a_coeff, V_x, x_librom, rank, resEQP); From 976f2bda68be9b46f8ccf482dffa04a751da8886 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 5 Jun 2023 21:35:42 +0200 Subject: [PATCH 012/120] Started eqp coefficient computation implementation And made fast calculation TODO --- .../prom/nonlinear_elasticity_global_rom.cpp | 5 +- .../nonlinear_elasticity_global_rom_eqp.hpp | 379 ++++++++++++++---- 2 files changed, 293 insertions(+), 91 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 0ff8c9747..49bfb7470 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1727,9 +1727,10 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const Vector resEQP; if (fastIntegration) - HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), eqp_qp, + //TODO + /* HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), eqp_qp, ir_eqp, &a_coeff, - x_librom, eqp_coef, resEQP); + x_librom, eqp_coef, resEQP); */ else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, &a_coeff, diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 0694ca8e3..821b69dbd 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -18,29 +18,275 @@ using namespace std; // Element matrix assembly, copying element loop from BilinearForm::Assemble(int skip_zeros) // and element matrix computation from HyperelasticNLFIntegrator::AssembleElementMatrix. void AssembleElementMatrix_HyperelasticNLFIntegrator(Coefficient *Q, - const FiniteElement &el, - ElementTransformation &Trans, - DenseMatrix &elmat) + const FiniteElement &el, + ElementTransformation &Trans, + DenseMatrix &elmat) { - + int dof = el.GetDof(), dim = el.GetDim(); + + DSh.SetSize(dof, dim); + DS.SetSize(dof, dim); + Jrt.SetSize(dim); + Jpt.SetSize(dim); + P.SetSize(dim); + PMatI.UseExternalData(elfun.GetData(), dof, dim); + elvect.SetSize(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); + + const IntegrationRule *ir = IntRule; + if (!ir) + { + ir = &(IntRules.Get(el.GetGeomType(), 2 * el.GetOrder() + 3)); // <--- + } + + elvect = 0.0; + model->SetTransformation(Ttr); + for (int i = 0; i < ir->GetNPoints(); i++) + { + const IntegrationPoint &ip = ir->IntPoint(i); + Ttr.SetIntPoint(&ip); + CalcInverse(Ttr.Jacobian(), Jrt); + + el.CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + MultAtB(PMatI, DS, Jpt); + + model->EvalP(Jpt, P); + + P *= ip.weight * Ttr.Weight(); + AddMultABt(DS, P, PMatO); + } } // Compute coefficients of the reduced integrator with respect to inputs Q and x // in HyperelasticNLFIntegrator_ComputeReducedEQP. void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, - std::vector const& rw, std::vector const& qp, - const IntegrationRule *ir, - CAROM::Matrix const& V, Vector & res) + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, + CAROM::Matrix const &V, Vector &res) { - + const int rdim = V.numColumns(); + MFEM_VERIFY(V.numRows() == fesR->GetTrueVSize(), ""); + MFEM_VERIFY(rw.size() == qp.size(), ""); + + const int ne = fesR->GetNE(); + const int nqe = ir->GetWeights().Size(); + + ElementTransformation *eltrans; + DofTransformation * doftrans; + const FiniteElement *fe = NULL; + Array vdofs; + + DenseMatrix trial_vshape; + + Vector Vx; + res.SetSize(rdim * rdim * rw.size()); + res = 0.0; + + // For the parallel case, we must get all DOFs of V on sampled elements. + CAROM::Matrix Vs; + + // Since V only has rows for true DOFs, we use a ParGridFunction to get all DOFs. + int eprev = -1; + int dof = 0; + int elemCount = 0; + + // First, find all sampled elements. + for (int i=0; iGetElementVDofs(e, vdofs); + if (dof > 0) + { + MFEM_VERIFY(dof == vdofs.Size(), "All elements must have same DOF size"); + } + dof = vdofs.Size(); + eprev = e; + elemCount++; + } + } + + // Now set Vs. + // Note that, ideally, these FOM data structures and operations should be + // avoided, but this only done in setup. + Vs.setSize(elemCount * dof, rdim); + ParGridFunction v_gf(fesR); + Vector vtrue(fesR->GetTrueVSize()); + } void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, - std::vector const& rw, std::vector const& qp, - const IntegrationRule *ir, Coefficient *Q, - CAROM::Matrix const& V, CAROM::Vector const& x, const int rank, Vector & res) + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, Coefficient *Q, + CAROM::Matrix const &V, CAROM::Vector const &x, const int rank, Vector &res) { - + + + const int rdim = V.numColumns(); + MFEM_VERIFY(V.numRows() == fesR->GetTrueVSize(), ""); + MFEM_VERIFY(rw.size() == qp.size(), ""); + + const int ne = fesR->GetNE(); + const int nqe = ir->GetWeights().Size(); + + ElementTransformation *eltrans; + DofTransformation * doftrans; + const FiniteElement *fe = NULL; + Array vdofs; + + DenseMatrix trial_vshape; + + Vector Vx; + res.SetSize(rdim * rdim * rw.size()); + res = 0.0; + + // For the parallel case, we must get all DOFs of V on sampled elements. + CAROM::Matrix Vs; + + // Since V only has rows for true DOFs, we use a ParGridFunction to get all DOFs. + int eprev = -1; + int dof = 0; + int elemCount = 0; + + // First, find all sampled elements. + for (int i=0; iGetElementVDofs(e, vdofs); + if (dof > 0) + { + MFEM_VERIFY(dof == vdofs.Size(), "All elements must have same DOF size"); + } + dof = vdofs.Size(); + eprev = e; + elemCount++; + } + } + + // Now set Vs. + // Note that, ideally, these FOM data structures and operations should be + // avoided, but this only done in setup. + Vs.setSize(elemCount * dof, rdim); + ParGridFunction v_gf(fesR); + Vector vtrue(fesR->GetTrueVSize()); + + for (int j=0; jGetElementVDofs(e, vdofs); + + for (int k=0; k= 0) ? vdofs[k] : -1 - vdofs[k]; + const double vk = (vdofs[k] >= 0) ? v_gf[dofk] : -v_gf[dofk]; + Vs((elemCount * dof) + k, j) = vk; + } + + eprev = e; + elemCount++; + } + } + } + + eprev = -1; + elemCount = 0; + int spaceDim = 0; + + for (int i=0; iIntPoint(qpi); + + if (e != eprev) // Update element transformation + { + doftrans = fesR->GetElementVDofs(e, vdofs); + fe = fesR->GetFE(e); + eltrans = fesR->GetElementTransformation(e); + + if (doftrans) + { + MFEM_ABORT("TODO"); + } + + dof = fe->GetDof(); + spaceDim = eltrans->GetSpaceDim(); + trial_vshape.SetSize(dof, spaceDim); + Vx.SetSize(spaceDim); + + eprev = e; + elemCount++; + } + + // Integrate at the current point + + eltrans->SetIntPoint(&ip); + // Ttr.SetIntPoint(&ip); From hyperelastic integrator + fe->CalcVShape(*eltrans, trial_vshape); + + double w = eltrans->Weight() * rw[i]; // using rw[i] instead of ip.weight + + /* CalcInverse(Ttr.Jacobian(), Jrt); + + el.CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + MultAtB(PMatI, DS, Jpt); + + model->EvalP(Jpt, P); + + P *= ip.weight * Ttr.Weight(); */ + + for (int jx=0; jx const& vdofs, - Coefficient *Q, Vector const& u, Vector const& v, - FiniteElement const& fe, ElementTransformation & Trans, Vector & r) +void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, + Coefficient *Q, Vector const &u, Vector const &v, + FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); int dof = fe.GetDof(); @@ -80,11 +326,11 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const& vdofs, u_i = 0.0; v_i = 0.0; - for (int j=0; j= 0) ? vdofs[j] : -1 - vdofs[j]; const double s = (vdofs[j] >= 0) ? 1.0 : -1.0; - for (int k=0; k const& vdofs, if (Q) { - w *= Q -> Eval (Trans, ip); + w *= Q->Eval(Trans, ip); } r[i] = 0.0; - for (int k=0; k const& vdofs, } } -// Precondition Gt by (V^T M V)^{-1} (of size NB x NB). -void PreconditionNNLS(ParFiniteElementSpace *fespace, - BilinearFormIntegrator *massInteg, - const CAROM::Matrix* B, - const int snapshot, - CAROM::Matrix & Gt) -{ - const int NB = B->numColumns(); - const int NQ = Gt.numRows(); - const int nsnap = Gt.numColumns() / NB; - - ParBilinearForm M(fespace); - - M.AddDomainIntegrator(massInteg); - M.Assemble(); - M.Finalize(); - HypreParMatrix *Mmat = M.ParallelAssemble(); - - CAROM::Matrix Mhat(NB, NB, false); - ComputeCtAB(*Mmat, *B, *B, Mhat); - Mhat.inverse(); - - CAROM::Vector PG(NB, false); - - for (int i = (snapshot >= 0 ? snapshot : 0); - i < (snapshot >= 0 ? snapshot+1 : nsnap); ++i) - { - for (int m=0; mGetNPoints(); const int ne = fespace_R->GetNE(); @@ -222,7 +423,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const int nsnap = BR_snapshots->numColumns(); MFEM_VERIFY(nsnap == BW_snapshots->numColumns() || - nsnap + nsets == BW_snapshots->numColumns(), ""); + nsnap + nsets == BW_snapshots->numColumns(), + ""); MFEM_VERIFY(BR->numRows() == BR_snapshots->numRows(), ""); MFEM_VERIFY(BR->numRows() == fespace_R->GetTrueVSize(), ""); MFEM_VERIFY(BW_snapshots->numRows() == fespace_W->GetTrueVSize(), ""); @@ -252,7 +454,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, skip = 1; } - for (int i=0; inumRows(); ++j) p_i[j] = (*BW_snapshots)(j, i + skip); @@ -274,7 +476,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, ParGridFunction vi_gf(fespace_R); vi_gf.SetFromTrueDofs(v_i); - for (int j=0; jnumRows(); ++k) v_j[k] = (*BR)(k, j); @@ -283,7 +485,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, vj_gf.SetFromTrueDofs(v_j); // TODO: is it better to make the element loop the outer loop? - for (int e=0; e vdofs; DofTransformation *doftrans = fespace_R->GetElementVDofs(e, vdofs); @@ -292,45 +494,44 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, ComputeElementRowOfG(ir0, vdofs, &a_coeff, vi_gf, vj_gf, fe, *eltrans, r); - for (int m=0; m const& w_el = ir0->GetWeights(); + Array const &w_el = ir0->GetWeights(); MFEM_VERIFY(w_el.Size() == nqe, ""); CAROM::Vector w(ne * nqe, true); - for (int i=0; i elements; Vector elemCount(pmesh->GetNE()); elemCount = 0.0; - for (int i=0; i 1.0e-12) { - const int e = i / nqe; // Element index + const int e = i / nqe; // Element index elements.insert(e); elemCount[e] += 1.0; } From 48adcfb91a71ebaf59ba5a504cf1329b82753c30 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 5 Jun 2023 22:28:17 +0200 Subject: [PATCH 013/120] Updated SetupEQP to make tests run --- .../prom/nonlinear_elasticity_global_rom.cpp | 2 +- .../nonlinear_elasticity_global_rom_eqp.hpp | 18 ++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 49bfb7470..4db31d5bf 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -912,7 +912,7 @@ int main(int argc, char *argv[]) { // EQP setup eqpSol = new CAROM::Vector(ir0->GetNPoints() * fespace.GetNE(), true); - SetupEQP_snapshots(ir0, myid, &fespace, &W_space, nsets, BR_librom, + SetupEQP_snapshots(ir0, myid, &fespace, nsets, H_librom, GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "H"), preconditionNNLS, tolNNLS, maxNNLSnnz, *eqpSol); diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 821b69dbd..1d126e398 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -405,13 +405,12 @@ void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, cout << rank << ": relative residual norm for NNLS solution of Gs = Gw: " << relNorm << endl; } + // Compute EQP solution from constraints on snapshots. void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, ParFiniteElementSpace *fespace_R, - ParFiniteElementSpace *fespace_W, const int nsets, const CAROM::Matrix *BR, const CAROM::Matrix *BR_snapshots, - const CAROM::Matrix *BW_snapshots, const bool precondition, const double nnls_tol, const int maxNNLSnnz, CAROM::Vector &sol) @@ -422,14 +421,13 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const int NQ = ne * nqe; const int nsnap = BR_snapshots->numColumns(); - MFEM_VERIFY(nsnap == BW_snapshots->numColumns() || - nsnap + nsets == BW_snapshots->numColumns(), + MFEM_VERIFY(nsnap == BR_snapshots->numColumns() || + nsnap + nsets == BR_snapshots->numColumns(), ""); MFEM_VERIFY(BR->numRows() == BR_snapshots->numRows(), ""); MFEM_VERIFY(BR->numRows() == fespace_R->GetTrueVSize(), ""); - MFEM_VERIFY(BW_snapshots->numRows() == fespace_W->GetTrueVSize(), ""); - const bool skipFirstW = (nsnap + nsets == BW_snapshots->numColumns()); + const bool skipFirstW = (nsnap + nsets == BR_snapshots->numColumns()); // Compute G of size (NB * nsnap) x NQ, but only store its transpose Gt. CAROM::Matrix Gt(NQ, NB * nsnap, true); @@ -440,7 +438,6 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, // with respect to the integration rule weight at that point, // where the "exact" quadrature solution is ir0->GetWeights(). - Vector p_i(BW_snapshots->numRows()); Vector v_i(BR_snapshots->numRows()); Vector v_j(BR->numRows()); @@ -456,9 +453,6 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, for (int i = 0; i < nsnap; ++i) { - for (int j = 0; j < BW_snapshots->numRows(); ++j) - p_i[j] = (*BW_snapshots)(j, i + skip); - for (int j = 0; j < BR_snapshots->numRows(); ++j) v_i[j] = (*BR_snapshots)(j, i); @@ -466,9 +460,9 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, skip++; // Set grid function for a(p) - ParGridFunction p_gf(fespace_W); + ParGridFunction p_gf(fespace_R); - p_gf.SetFromTrueDofs(p_i); + p_gf.SetFromTrueDofs(v_i); GridFunctionCoefficient p_coeff(&p_gf); TransformedCoefficient a_coeff(&p_coeff, NonlinearCoefficient); From 954c2a7cb88b2046f8e9489895173267b98196e2 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 5 Jun 2023 22:38:29 +0200 Subject: [PATCH 014/120] Some more fixes for the compiler --- .../prom/nonlinear_elasticity_global_rom.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 4db31d5bf..4dfa7ecc2 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -114,6 +114,8 @@ using namespace std; using namespace mfem; +#include "nonlinear_elasticity_global_rom_eqp.hpp" + class ReducedSystemOperator; class HyperelasticOperator : public TimeDependentOperator @@ -197,7 +199,7 @@ class RomOperator : public TimeDependentOperator std::vector eqp_rw; std::vector eqp_qp; Vector eqp_coef; - // const bool fastIntegration = true; //TODO: implement fast integration + const bool fastIntegration = false; //TODO: implement fast integration int rank; @@ -1084,10 +1086,17 @@ int main(int argc, char *argv[]) } } - if (myid == 0 && !use_eqp) + if (myid == 0) { + if (!use_eqp) + { // Define operator in sample space soper = new HyperelasticOperator(*sp_XV_space, ess_tdof_list_sp, visc, mu, K); + } + else + { + soper = new HyperelasticOperator(*fespace, ess_tdof_list, visc, mu, K); + } } if (hyperreduce) // TODO: ask about whether this is needed. @@ -1723,14 +1732,17 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const if (eqp) { // Lift v-vector and save V_v.mult(v_librom, *z_v_librom); - V_x->transposeMult(*z_v_librom, dx_dt_librom); + V_x.transposeMult(*z_v_librom, dx_dt_librom); Vector resEQP; if (fastIntegration) //TODO + { /* HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), eqp_qp, ir_eqp, &a_coeff, x_librom, eqp_coef, resEQP); */ + } + else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, &a_coeff, From 368a705ff06f23189630f4ae27fd063d99a170ee Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 5 Jun 2023 22:43:05 +0200 Subject: [PATCH 015/120] Changed rrdim -> rxdim --- examples/prom/nonlinear_elasticity_global_rom.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 4dfa7ecc2..774c6bb80 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1758,10 +1758,10 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const // In the EQP case, there is no use of a ROM space for the nonlinear // term. Instead, the FOM computation of the nonlinear term is // approximated by the reduced quadrature rule in the FOM space. - // Therefore, the residual here is of dimension rrdim. + // Therefore, the residual here is of dimension rxdim. - MFEM_VERIFY(resEQP.Size() == rrdim, ""); - for (int i = 0; i < rrdim; ++i) + MFEM_VERIFY(resEQP.Size() == rxdim, ""); + for (int i = 0; i < rxdim; ++i) z_librom[i] += resEQP[i]; } else From 0a6e7f6a62da973268824477f982605d3fb0f95f Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 8 Jun 2023 22:56:01 +0200 Subject: [PATCH 016/120] Started SetupEqp_snapshots for hyperelastic nonlinear operator --- .../prom/nonlinear_elasticity_global_rom.cpp | 8 +- .../nonlinear_elasticity_global_rom_eqp.hpp | 192 ++++++++++-------- 2 files changed, 113 insertions(+), 87 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 774c6bb80..7b9670e56 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -912,11 +912,12 @@ int main(int argc, char *argv[]) if (use_eqp) { + NeoHookeanmodel = new NeoHookeanModel(mu, K); // EQP setup eqpSol = new CAROM::Vector(ir0->GetNPoints() * fespace.GetNE(), true); SetupEQP_snapshots(ir0, myid, &fespace, nsets, H_librom, GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "H"), - preconditionNNLS, tolNNLS, maxNNLSnnz, *eqpSol); + preconditionNNLS, tolNNLS, maxNNLSnnz, model, *eqpSol); if (writeSampleMesh) WriteMeshEQP(pmesh, myid, ir0->GetNPoints(), *eqpSol); @@ -1736,11 +1737,8 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const Vector resEQP; if (fastIntegration) - //TODO { - /* HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), eqp_qp, - ir_eqp, &a_coeff, - x_librom, eqp_coef, resEQP); */ + MFEM_ABORT("TODO"); } else diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 1d126e398..d8560d734 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -65,7 +65,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, const IntegrationRule *ir, CAROM::Matrix const &V, Vector &res) { - const int rdim = V.numColumns(); + const int rdim = V.numColumns(); MFEM_VERIFY(V.numRows() == fesR->GetTrueVSize(), ""); MFEM_VERIFY(rw.size() == qp.size(), ""); @@ -73,7 +73,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, const int nqe = ir->GetWeights().Size(); ElementTransformation *eltrans; - DofTransformation * doftrans; + DofTransformation *doftrans; const FiniteElement *fe = NULL; Array vdofs; @@ -92,11 +92,11 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, int elemCount = 0; // First, find all sampled elements. - for (int i=0; iGetElementVDofs(e, vdofs); if (dof > 0) @@ -115,7 +115,6 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, Vs.setSize(elemCount * dof, rdim); ParGridFunction v_gf(fesR); Vector vtrue(fesR->GetTrueVSize()); - } void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, @@ -124,7 +123,6 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, CAROM::Matrix const &V, CAROM::Vector const &x, const int rank, Vector &res) { - const int rdim = V.numColumns(); MFEM_VERIFY(V.numRows() == fesR->GetTrueVSize(), ""); MFEM_VERIFY(rw.size() == qp.size(), ""); @@ -133,7 +131,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, const int nqe = ir->GetWeights().Size(); ElementTransformation *eltrans; - DofTransformation * doftrans; + DofTransformation *doftrans; const FiniteElement *fe = NULL; Array vdofs; @@ -152,11 +150,11 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, int elemCount = 0; // First, find all sampled elements. - for (int i=0; iGetElementVDofs(e, vdofs); if (dof > 0) @@ -176,25 +174,25 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, ParGridFunction v_gf(fesR); Vector vtrue(fesR->GetTrueVSize()); - for (int j=0; jGetElementVDofs(e, vdofs); - for (int k=0; k= 0) ? vdofs[k] : -1 - vdofs[k]; const double vk = (vdofs[k] >= 0) ? v_gf[dofk] : -v_gf[dofk]; @@ -211,14 +209,14 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, elemCount = 0; int spaceDim = 0; - for (int i=0; iIntPoint(qpi); - if (e != eprev) // Update element transformation + if (e != eprev) // Update element transformation { doftrans = fesR->GetElementVDofs(e, vdofs); fe = fesR->GetFE(e); @@ -256,27 +254,27 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, P *= ip.weight * Ttr.Weight(); */ - for (int jx=0; jx const &vdofs, - Coefficient *Q, Vector const &u, Vector const &v, + Vector const &h, Vector const &v, const Vector &elfun, // Q is elfun == h? or elfun == u? + NeoHookeanmodel model, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); - int dof = fe.GetDof(); + int dof = fe.GetDof(); // Get number of dofs in element int spaceDim = Trans.GetSpaceDim(); Vector u_i(spaceDim); @@ -313,42 +310,72 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, DenseMatrix trial_vshape(dof, spaceDim); + // Initialize nonlinear operator matrices (there is probably a better way) + int dim = el.GetDim(); + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DenseMatrix Jpt(dim); + DenseMatrix P(dim); + DenseMatrix PMatI; + PMatI.UseExternalData(elfun.GetData(), dof, dim); + // PMatO.SetSize(dof, dim); // Q: should this be here? + + // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) { + // Get integration point const IntegrationPoint &ip = ir->IntPoint(i); + // Set integration point in the element transformation Trans.SetIntPoint(&ip); - fe.CalcVShape(Trans, trial_vshape); + // Evaluate the element shape functions at the integration point + fe.CalcVShape(Trans, trial_vshape); // Q: Should this actually be CalcDShape? - double w = Trans.Weight(); + // Get the transformation weight + double t = Trans.Weight(); - u_i = 0.0; + // Initialize h_i and v_i + h_i = 0.0; v_i = 0.0; + // For every dof in element for (int j = 0; j < dof; ++j) { + // Q: Why can vdofs[j] < 0 be true? const int dofj = (vdofs[j] >= 0) ? vdofs[j] : -1 - vdofs[j]; const double s = (vdofs[j] >= 0) ? 1.0 : -1.0; + + // Calculate h_i = B_e * h_e, where h_e are the samples + // for this snapshot at this element dofs + // Also calculate v_i = B_e * v_e for (int k = 0; k < spaceDim; ++k) { - u_i[k] += s * u[dofj] * trial_vshape(j, k); + h_i[k] += s * h[dofj] * trial_vshape(j, k); v_i[k] += s * v[dofj] * trial_vshape(j, k); } } + + // Compute action of nonlinear operator + CalcInverse(Trans.Jacobian(), Jrt); + fe.CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P); - if (Q) - { - w *= Q->Eval(Trans, ip); - } + // AddMultABt(DS, P, PMatO); //Q: Should this be here? r[i] = 0.0; - for (int k = 0; k < spaceDim; ++k) - { - r[i] += u_i[k] * v_i[k]; - } - r[i] *= w; + // Calculate r[i] = v_i^T * P * h_i + // P is a 2x2 matrix + // Perform the vector-matrix-vector multiplication: a^T * B * c + Vector temp(2); + P.Mult(h_i, temp); + double result = v_i * temp; + // Scale by element transformation + r[i] *= t; } } @@ -405,29 +432,28 @@ void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, cout << rank << ": relative residual norm for NNLS solution of Gs = Gw: " << relNorm << endl; } - // Compute EQP solution from constraints on snapshots. void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, - ParFiniteElementSpace *fespace_R, - const int nsets, const CAROM::Matrix *BR, - const CAROM::Matrix *BR_snapshots, + ParFiniteElementSpace *fespace_H, + const int nsets, const CAROM::Matrix *BH, + const CAROM::Matrix *BH_snapshots, const bool precondition, const double nnls_tol, - const int maxNNLSnnz, + const int maxNNLSnnz, NeoHookeanmodel model, CAROM::Vector &sol) { const int nqe = ir0->GetNPoints(); - const int ne = fespace_R->GetNE(); - const int NB = BR->numColumns(); + const int ne = fespace_H->GetNE(); + const int NB = BH->numColumns(); const int NQ = ne * nqe; - const int nsnap = BR_snapshots->numColumns(); + const int nsnap = BH_snapshots->numColumns(); - MFEM_VERIFY(nsnap == BR_snapshots->numColumns() || - nsnap + nsets == BR_snapshots->numColumns(), + MFEM_VERIFY(nsnap == BH_snapshots->numColumns() || + nsnap + nsets == BH_snapshots->numColumns(), // Q: nsets? ""); - MFEM_VERIFY(BR->numRows() == BR_snapshots->numRows(), ""); - MFEM_VERIFY(BR->numRows() == fespace_R->GetTrueVSize(), ""); + MFEM_VERIFY(BH->numRows() == BH_snapshots->numRows(), ""); + MFEM_VERIFY(BH->numRows() == fespace_H->GetTrueVSize(), ""); - const bool skipFirstW = (nsnap + nsets == BR_snapshots->numColumns()); + const bool skipFirstW = (nsnap + nsets == BH_snapshots->numColumns()); // Compute G of size (NB * nsnap) x NQ, but only store its transpose Gt. CAROM::Matrix Gt(NQ, NB * nsnap, true); @@ -438,8 +464,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, // with respect to the integration rule weight at that point, // where the "exact" quadrature solution is ir0->GetWeights(). - Vector v_i(BR_snapshots->numRows()); - Vector v_j(BR->numRows()); + Vector h_i(BH_snapshots->numRows()); + Vector v_j(BH->numRows()); Vector r(nqe); @@ -451,42 +477,44 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, skip = 1; } + // For every snapshot for (int i = 0; i < nsnap; ++i) { - for (int j = 0; j < BR_snapshots->numRows(); ++j) - v_i[j] = (*BR_snapshots)(j, i); + // Set the sampled dofs from the snapshot matrix + for (int j = 0; j < BH_snapshots->numRows(); ++j) + h_i[j] = (*BH_snapshots)(j, i); + // Q: Not sure what this does if (skipFirstW && i > 0 && i % nsnapPerSet == 0) skip++; - // Set grid function for a(p) - ParGridFunction p_gf(fespace_R); - - p_gf.SetFromTrueDofs(v_i); - - GridFunctionCoefficient p_coeff(&p_gf); - TransformedCoefficient a_coeff(&p_coeff, NonlinearCoefficient); - - ParGridFunction vi_gf(fespace_R); - vi_gf.SetFromTrueDofs(v_i); + // Set grid function for h_i + ParGridFunction hi_gf(fespace_H); + hi_gf.SetFromTrueDofs(h_i); + // For each basis vector for (int j = 0; j < NB; ++j) { - for (int k = 0; k < BR->numRows(); ++k) - v_j[k] = (*BR)(k, j); + // Get basis vector + for (int k = 0; k < BH->numRows(); ++k) + v_j[k] = (*BH)(k, j); - ParGridFunction vj_gf(fespace_R); + // Set grid function for basis vector + ParGridFunction vj_gf(fespace_H); vj_gf.SetFromTrueDofs(v_j); // TODO: is it better to make the element loop the outer loop? + // For each element for (int e = 0; e < ne; ++e) { + // Get element and its dofs and transformation. Array vdofs; - DofTransformation *doftrans = fespace_R->GetElementVDofs(e, vdofs); - const FiniteElement &fe = *fespace_R->GetFE(e); - ElementTransformation *eltrans = fespace_R->GetElementTransformation(e); + DofTransformation *doftrans = fespace_H->GetElementVDofs(e, vdofs); + const FiniteElement &fe = *fespace_H->GetFE(e); + ElementTransformation *eltrans = fespace_H->GetElementTransformation(e); - ComputeElementRowOfG(ir0, vdofs, &a_coeff, vi_gf, vj_gf, fe, *eltrans, r); + // Compute the row of G corresponding to element e, store in r + ComputeElementRowOfG(ir0, vdofs, &a_coeff, hi_gf, vj_gf, model, fe, *eltrans, r); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; @@ -495,7 +523,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, if (precondition) { - // TODO + MFEM_ABORT("TODO"); } } // Loop (i) over snapshots From 1353c0a5f4ba6bcd520c89128a86e8d33a8c055e Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 9 Jun 2023 01:36:32 +0200 Subject: [PATCH 017/120] Added sketch for compute reduced eqp --- .../nonlinear_elasticity_global_rom_eqp.hpp | 198 +++++++++++------- 1 file changed, 124 insertions(+), 74 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index d8560d734..fcde43cc2 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -15,61 +15,29 @@ using namespace mfem; using namespace std; -// Element matrix assembly, copying element loop from BilinearForm::Assemble(int skip_zeros) -// and element matrix computation from HyperelasticNLFIntegrator::AssembleElementMatrix. -void AssembleElementMatrix_HyperelasticNLFIntegrator(Coefficient *Q, - const FiniteElement &el, - ElementTransformation &Trans, - DenseMatrix &elmat) -{ - int dof = el.GetDof(), dim = el.GetDim(); - - DSh.SetSize(dof, dim); - DS.SetSize(dof, dim); - Jrt.SetSize(dim); - Jpt.SetSize(dim); - P.SetSize(dim); - PMatI.UseExternalData(elfun.GetData(), dof, dim); - elvect.SetSize(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); - - const IntegrationRule *ir = IntRule; - if (!ir) - { - ir = &(IntRules.Get(el.GetGeomType(), 2 * el.GetOrder() + 3)); // <--- - } - - elvect = 0.0; - model->SetTransformation(Ttr); - for (int i = 0; i < ir->GetNPoints(); i++) - { - const IntegrationPoint &ip = ir->IntPoint(i); - Ttr.SetIntPoint(&ip); - CalcInverse(Ttr.Jacobian(), Jrt); - - el.CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - MultAtB(PMatI, DS, Jpt); - - model->EvalP(Jpt, P); - - P *= ip.weight * Ttr.Weight(); - AddMultABt(DS, P, PMatO); - } -} - // Compute coefficients of the reduced integrator with respect to inputs Q and x // in HyperelasticNLFIntegrator_ComputeReducedEQP. void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, CAROM::Matrix const &V, Vector &res) +{ + MFEM_ABORT("TODO") +} + +void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, Coefficient *Q, + CAROM::Matrix const &V, CAROM::Vector const &x, const int rank, Vector &res) { const int rdim = V.numColumns(); - MFEM_VERIFY(V.numRows() == fesR->GetTrueVSize(), ""); MFEM_VERIFY(rw.size() == qp.size(), ""); + MFEM_VERIFY(x.dim() == rdim, ""); + MFEM_VERIFY(V.numRows() == fesR->GetTrueVSize(), ""); + + MFEM_VERIFY(rank == 0, + "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); - const int ne = fesR->GetNE(); const int nqe = ir->GetWeights().Size(); ElementTransformation *eltrans; @@ -80,41 +48,115 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, DenseMatrix trial_vshape; Vector Vx; - res.SetSize(rdim * rdim * rw.size()); + Vector Vj; + res.SetSize(rdim); res = 0.0; - // For the parallel case, we must get all DOFs of V on sampled elements. - CAROM::Matrix Vs; - - // Since V only has rows for true DOFs, we use a ParGridFunction to get all DOFs. int eprev = -1; int dof = 0; - int elemCount = 0; + int spaceDim = 0; + + // Note that the alternative version VectorFEMassIntegrator_ComputeReducedEQP_Fast + // of this function is optimized by storing some intermediate computations. - // First, find all sampled elements. for (int i = 0; i < rw.size(); ++i) { const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); if (e != eprev) // Update element transformation { doftrans = fesR->GetElementVDofs(e, vdofs); - if (dof > 0) + fe = fesR->GetFE(e); + eltrans = fesR->GetElementTransformation(e); + + if (doftrans) { - MFEM_VERIFY(dof == vdofs.Size(), "All elements must have same DOF size"); + MFEM_ABORT("TODO"); } - dof = vdofs.Size(); + + dof = fe->GetDof(); + spaceDim = eltrans->GetSpaceDim(); + trial_vshape.SetSize(dof, spaceDim); + Vx.SetSize(spaceDim); + Vj.SetSize(spaceDim); + + // Initialize nonlinear operator matrices (there is probably a better way) + int dim = fe.GetDim(); + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DenseMatrix Jpt(dim); + DenseMatrix P(dim); + DenseMatrix PMatI; + PMatI.UseExternalData(elfun.GetData(), dof, dim); // Q: How to replace? + // PMatO.SetSize(dof, dim); // Q: should this be here? + eprev = e; - elemCount++; } - } - // Now set Vs. - // Note that, ideally, these FOM data structures and operations should be - // avoided, but this only done in setup. - Vs.setSize(elemCount * dof, rdim); - ParGridFunction v_gf(fesR); - Vector vtrue(fesR->GetTrueVSize()); + // Integrate at the current point + eltrans->SetIntPoint(&ip); + fe->CalcVShape(*eltrans, trial_vshape); + + double w = eltrans->Weight() * rw[i]; // using rw[i] instead of ip.weight + + // Compute action of nonlinear operator + CalcInverse(eltrans.Jacobian(), Jrt); + fe.CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P); + // AddMultABt(DS, P, PMatO); //Q: Should this be here? + + // Lift Vx at ip. + Vx = 0.0; + for (int k = 0; k < dof; ++k) + { + const int dofk = (vdofs[k] >= 0) ? vdofs[k] : -1 - vdofs[k]; + + double Vx_k = 0.0; + for (int j = 0; j < rdim; ++j) + { + Vx_k += V(dofk, j) * x(j); + } + + if (vdofs[k] < 0) // Q:why this? + Vx_k = -Vx_k; + + for (int j = 0; j < spaceDim; ++j) + Vx[j] += Vx_k * trial_vshape(k, j); + } + + Vj = 0.0; + for (int k = 0; k < spaceDim; ++k) + { + double Vjk = 0.0; + for (int l = 0; l < dof; ++l) + { + // Q: Should these ones be the same? + const int dofl = (vdofs[l] >= 0) ? vdofs[l] : -1 - vdofs[l]; + const double s = (vdofs[l] >= 0) ? 1.0 : -1.0; + Vjk += s * V(dofl, j) * trial_vshape(l, k); + } + + Vj[k] = Vjk; + } + + // Calculate r[i] = Vj^T * P * Vx + // Perform the vector-matrix-vector multiplication: a^T * B * c + Vector temp(dim); + P.Mult(Vx, temp); + w *= Vj * temp; + + // Perform final scaling + for (int j = 0; j < rdim; ++j) + { + res[j] += w + } + } } void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, @@ -193,7 +235,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, doftrans = fesR->GetElementVDofs(e, vdofs); for (int k = 0; k < dof; ++k) - { + { // Q: Why this code? const int dofk = (vdofs[k] >= 0) ? vdofs[k] : -1 - vdofs[k]; const double vk = (vdofs[k] >= 0) ? v_gf[dofk] : -v_gf[dofk]; Vs((elemCount * dof) + k, j) = vk; @@ -232,6 +274,17 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, trial_vshape.SetSize(dof, spaceDim); Vx.SetSize(spaceDim); + // Initialize nonlinear operator matrices (there is probably a better way) + int dim = el.GetDim(); + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DenseMatrix Jpt(dim); + DenseMatrix P(dim); + DenseMatrix PMatI; + PMatI.UseExternalData(elfun.GetData(), dof, dim); // Q: How to replace? + // PMatO.SetSize(dof, dim); // Q: should this be here? + eprev = e; elemCount++; } @@ -239,20 +292,18 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, // Integrate at the current point eltrans->SetIntPoint(&ip); - // Ttr.SetIntPoint(&ip); From hyperelastic integrator fe->CalcVShape(*eltrans, trial_vshape); double w = eltrans->Weight() * rw[i]; // using rw[i] instead of ip.weight - /* CalcInverse(Ttr.Jacobian(), Jrt); - - el.CalcDShape(ip, DSh); + // Compute action of nonlinear operator + CalcInverse(eltrans.Jacobian(), Jrt); + fe.CalcDShape(ip, DSh); Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); - P *= ip.weight * Ttr.Weight(); */ + // AddMultABt(DS, P, PMatO); //Q: Should this be here? for (int jx = 0; jx < rdim; ++jx) { @@ -356,7 +407,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, v_i[k] += s * v[dofj] * trial_vshape(j, k); } } - + // Compute action of nonlinear operator CalcInverse(Trans.Jacobian(), Jrt); fe.CalcDShape(ip, DSh); @@ -369,11 +420,10 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, r[i] = 0.0; // Calculate r[i] = v_i^T * P * h_i - // P is a 2x2 matrix // Perform the vector-matrix-vector multiplication: a^T * B * c - Vector temp(2); + Vector temp(dim); P.Mult(h_i, temp); - double result = v_i * temp; + r[i] += v_i * temp; // Scale by element transformation r[i] *= t; } From d5ba149fb0beda6a914c0d066288181f5faa348f Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 9 Jun 2023 22:03:27 +0200 Subject: [PATCH 018/120] Code compiles --- .../prom/nonlinear_elasticity_global_rom.cpp | 19 ++++--- .../nonlinear_elasticity_global_rom_eqp.hpp | 50 ++++++++++--------- 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 7b9670e56..2f7b9e6d3 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -103,6 +103,7 @@ #include "linalg/Vector.h" #include "linalg/BasisGenerator.h" #include "linalg/BasisReader.h" +#include "linalg/NNLS.h" #include "hyperreduction/Hyperreduction.h" #include "mfem/SampleMesh.hpp" #include "mfem/Utilities.hpp" @@ -912,12 +913,12 @@ int main(int argc, char *argv[]) if (use_eqp) { - NeoHookeanmodel = new NeoHookeanModel(mu, K); + NeoHookeanModel* neo_model = new NeoHookeanModel(mu, K); // EQP setup eqpSol = new CAROM::Vector(ir0->GetNPoints() * fespace.GetNE(), true); SetupEQP_snapshots(ir0, myid, &fespace, nsets, H_librom, GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "H"), - preconditionNNLS, tolNNLS, maxNNLSnnz, model, *eqpSol); + preconditionNNLS, tolNNLS, maxNNLSnnz, neo_model, *eqpSol); if (writeSampleMesh) WriteMeshEQP(pmesh, myid, ir0->GetNPoints(), *eqpSol); @@ -984,6 +985,7 @@ int main(int argc, char *argv[]) Vector *w_x0 = 0; int sp_size = 0; + Array ess_tdof_list_sp; // Initialize sample essential boundary conditions if (myid == 0 && !use_eqp) { @@ -1072,8 +1074,8 @@ int main(int argc, char *argv[]) } } - // Initialize essential dof list in sampling space - Array ess_tdof_list_sp(num_ess_sp); + // Set essential dof list in sampling space size + ess_tdof_list_sp.SetSize(num_ess_sp); // Add indices to list int ctr = 0; @@ -1096,7 +1098,7 @@ int main(int argc, char *argv[]) } else { - soper = new HyperelasticOperator(*fespace, ess_tdof_list, visc, mu, K); + soper = new HyperelasticOperator(fespace, ess_tdof_list, visc, mu, K); } } @@ -1143,6 +1145,7 @@ int main(int argc, char *argv[]) } ode_solver->Init(*romop); + } else { @@ -1705,7 +1708,7 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, << fom->fespace.GetNE() << endl; // TODO: implement the one below - GetEQPCoefficients_HyperelasticNLFIntegrator(); + GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, *U_H, eqp_coef); } } @@ -1743,7 +1746,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, - eqp_qp, ir_eqp, &a_coeff, + eqp_qp, ir_eqp, V_x, x_librom, rank, resEQP); Vector recv(resEQP); @@ -1760,7 +1763,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const MFEM_VERIFY(resEQP.Size() == rxdim, ""); for (int i = 0; i < rxdim; ++i) - z_librom[i] += resEQP[i]; + z[i] += resEQP[i]; } else { // Lift x- and v-vector diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index fcde43cc2..952631695 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -9,8 +9,8 @@ *****************************************************************************/ // Functions used by mixed_nonlinear_diffusion.cpp with EQP. - #include "mfem/Utilities.hpp" +//#include "fem/nonlininteg.hpp" using namespace mfem; using namespace std; @@ -25,10 +25,13 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, MFEM_ABORT("TODO") } + void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, Coefficient *Q, + const IntegrationRule *ir, CAROM::Matrix const &V, CAROM::Vector const &x, const int rank, Vector &res) +{MFEM_ABORT("TODO");} +/* { const int rdim = V.numColumns(); MFEM_VERIFY(rw.size() == qp.size(), ""); @@ -84,7 +87,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, Vj.SetSize(spaceDim); // Initialize nonlinear operator matrices (there is probably a better way) - int dim = fe.GetDim(); + int dim = fe->GetDim(); DenseMatrix DSh(dof, dim); DenseMatrix DS(dof, dim); DenseMatrix Jrt(dim); @@ -104,7 +107,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, double w = eltrans->Weight() * rw[i]; // using rw[i] instead of ip.weight // Compute action of nonlinear operator - CalcInverse(eltrans.Jacobian(), Jrt); + CalcInverse(eltrans->Jacobian(), Jrt); fe.CalcDShape(ip, DSh); Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); @@ -158,7 +161,8 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } } } - +*/ +/* void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, Coefficient *Q, @@ -337,6 +341,8 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } } +*/ + /* TODO if time... void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, std::vector const& qp, const IntegrationRule *ir, @@ -348,29 +354,29 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes */ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, - Vector const &h, Vector const &v, const Vector &elfun, // Q is elfun == h? or elfun == u? - NeoHookeanmodel model, + Vector const &h, Vector const &v, + NeoHookeanModel *model, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); int dof = fe.GetDof(); // Get number of dofs in element int spaceDim = Trans.GetSpaceDim(); - Vector u_i(spaceDim); + Vector h_i(spaceDim); Vector v_i(spaceDim); DenseMatrix trial_vshape(dof, spaceDim); // Initialize nonlinear operator matrices (there is probably a better way) - int dim = el.GetDim(); + int dim = fe.GetDim(); DenseMatrix DSh(dof, dim); DenseMatrix DS(dof, dim); DenseMatrix Jrt(dim); DenseMatrix Jpt(dim); DenseMatrix P(dim); - DenseMatrix PMatI; - PMatI.UseExternalData(elfun.GetData(), dof, dim); - // PMatO.SetSize(dof, dim); // Q: should this be here? + DenseMatrix PMatI(dof, dim); // Extract element dofs + DenseMatrix PMatO(dof, dim); + Vector el_vect(dof * dim); // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) @@ -382,7 +388,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Trans.SetIntPoint(&ip); // Evaluate the element shape functions at the integration point - fe.CalcVShape(Trans, trial_vshape); // Q: Should this actually be CalcDShape? + fe.CalcVShape(Trans, trial_vshape); // Get the transformation weight double t = Trans.Weight(); @@ -403,10 +409,11 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, // Also calculate v_i = B_e * v_e for (int k = 0; k < spaceDim; ++k) { - h_i[k] += s * h[dofj] * trial_vshape(j, k); + //h_i[k] += s * h[dofj] * trial_vshape(j, k); v_i[k] += s * v[dofj] * trial_vshape(j, k); + // PMatI[j, k] = s * h[dofj]; } - } + } // Compute action of nonlinear operator CalcInverse(Trans.Jacobian(), Jrt); @@ -414,16 +421,13 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); model->EvalP(Jpt, P); - - // AddMultABt(DS, P, PMatO); //Q: Should this be here? + AddMultABt(DS, P, PMatO); r[i] = 0.0; - // Calculate r[i] = v_i^T * P * h_i + // Calculate r[i] = v_i^T * el_vect // Perform the vector-matrix-vector multiplication: a^T * B * c - Vector temp(dim); - P.Mult(h_i, temp); - r[i] += v_i * temp; + r[i] += v_i * el_vect; // Scale by element transformation r[i] *= t; } @@ -488,7 +492,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const int nsets, const CAROM::Matrix *BH, const CAROM::Matrix *BH_snapshots, const bool precondition, const double nnls_tol, - const int maxNNLSnnz, NeoHookeanmodel model, + const int maxNNLSnnz, NeoHookeanModel *model, CAROM::Vector &sol) { const int nqe = ir0->GetNPoints(); @@ -564,7 +568,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, ElementTransformation *eltrans = fespace_H->GetElementTransformation(e); // Compute the row of G corresponding to element e, store in r - ComputeElementRowOfG(ir0, vdofs, &a_coeff, hi_gf, vj_gf, model, fe, *eltrans, r); + ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, model, fe, *eltrans, r); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; From a799238437cb72c58869c29a6d66f7db2cb57057 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Sat, 10 Jun 2023 22:41:07 +0200 Subject: [PATCH 019/120] Added element function to G row computation --- .../nonlinear_elasticity_global_rom_eqp.hpp | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 952631695..9e2ffc973 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -10,7 +10,7 @@ // Functions used by mixed_nonlinear_diffusion.cpp with EQP. #include "mfem/Utilities.hpp" -//#include "fem/nonlininteg.hpp" +// #include "fem/nonlininteg.hpp" using namespace mfem; using namespace std; @@ -25,12 +25,13 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, MFEM_ABORT("TODO") } - void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, CAROM::Matrix const &V, CAROM::Vector const &x, const int rank, Vector &res) -{MFEM_ABORT("TODO");} +{ + MFEM_ABORT("TODO"); +} /* { const int rdim = V.numColumns(); @@ -138,7 +139,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, { double Vjk = 0.0; for (int l = 0; l < dof; ++l) - { + { // Q: Should these ones be the same? const int dofl = (vdofs[l] >= 0) ? vdofs[l] : -1 - vdofs[l]; const double s = (vdofs[l] >= 0) ? 1.0 : -1.0; @@ -162,7 +163,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } } */ -/* +/* void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, Coefficient *Q, @@ -355,7 +356,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &h, Vector const &v, - NeoHookeanModel *model, + NeoHookeanModel *model, Vector elfun, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); @@ -374,7 +375,8 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, DenseMatrix Jrt(dim); DenseMatrix Jpt(dim); DenseMatrix P(dim); - DenseMatrix PMatI(dof, dim); // Extract element dofs + DenseMatrix PMatI; // Extract element dofs + PMatI.UseExternalData(elfun.GetData(), dof, dim); DenseMatrix PMatO(dof, dim); Vector el_vect(dof * dim); @@ -388,7 +390,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Trans.SetIntPoint(&ip); // Evaluate the element shape functions at the integration point - fe.CalcVShape(Trans, trial_vshape); + fe.CalcVShape(Trans, trial_vshape); // Get the transformation weight double t = Trans.Weight(); @@ -409,11 +411,11 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, // Also calculate v_i = B_e * v_e for (int k = 0; k < spaceDim; ++k) { - //h_i[k] += s * h[dofj] * trial_vshape(j, k); + // h_i[k] += s * h[dofj] * trial_vshape(j, k); v_i[k] += s * v[dofj] * trial_vshape(j, k); // PMatI[j, k] = s * h[dofj]; } - } + } // Compute action of nonlinear operator CalcInverse(Trans.Jacobian(), Jrt); @@ -425,7 +427,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, r[i] = 0.0; - // Calculate r[i] = v_i^T * el_vect + // Calculate r[i] = v_i^T * el_vect // Perform the vector-matrix-vector multiplication: a^T * B * c r[i] += v_i * el_vect; // Scale by element transformation @@ -531,6 +533,9 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, skip = 1; } + // Get prolongation matrix + const Operator* P = fespace_H->GetProlongationMatrix(); + // For every snapshot for (int i = 0; i < nsnap; ++i) { @@ -538,6 +543,22 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, for (int j = 0; j < BH_snapshots->numRows(); ++j) h_i[j] = (*BH_snapshots)(j, i); + // Get prolongated dofs + // See: https://docs.mfem.org/html/nonlinearform_8cpp_source.html#l00131 + Vector ph_i; + Vector elfun; + if (P) + { + ph_i.SetSize(P->Height()); + P->Mult(h_i, ph_i); + } + else + { + ph_i.SetSize(h_i.Size()); + for (int j = 0; j < h_i.Size(); ++j) + ph_i[j] = h_i[j]; + } + // Q: Not sure what this does if (skipFirstW && i > 0 && i % nsnapPerSet == 0) skip++; @@ -566,9 +587,10 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, DofTransformation *doftrans = fespace_H->GetElementVDofs(e, vdofs); const FiniteElement &fe = *fespace_H->GetFE(e); ElementTransformation *eltrans = fespace_H->GetElementTransformation(e); + ph_i.GetSubVector(vdofs, elfun); // Compute the row of G corresponding to element e, store in r - ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, model, fe, *eltrans, r); + ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, model, elfun, fe, *eltrans, r); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; From f7428306972aa35bb2d4f66bdfc78560ee96632a Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:42:25 +0200 Subject: [PATCH 020/120] Testing element vector error --- .../prom/nonlinear_elasticity_global_rom.cpp | 7 +- .../nonlinear_elasticity_global_rom_eqp.hpp | 74 ++++++++++++++++++- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 2f7b9e6d3..59de61714 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -119,7 +119,7 @@ using namespace mfem; class ReducedSystemOperator; -class HyperelasticOperator : public TimeDependentOperator +/* class HyperelasticOperator : public TimeDependentOperator { protected: @@ -154,7 +154,7 @@ class HyperelasticOperator : public TimeDependentOperator HypreParMatrix Smat; virtual ~HyperelasticOperator(); -}; +}; */ class RomOperator : public TimeDependentOperator { @@ -913,12 +913,11 @@ int main(int argc, char *argv[]) if (use_eqp) { - NeoHookeanModel* neo_model = new NeoHookeanModel(mu, K); // EQP setup eqpSol = new CAROM::Vector(ir0->GetNPoints() * fespace.GetNE(), true); SetupEQP_snapshots(ir0, myid, &fespace, nsets, H_librom, GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "H"), - preconditionNNLS, tolNNLS, maxNNLSnnz, neo_model, *eqpSol); + preconditionNNLS, tolNNLS, maxNNLSnnz, oper, *eqpSol); if (writeSampleMesh) WriteMeshEQP(pmesh, myid, ir0->GetNPoints(), *eqpSol); diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 9e2ffc973..577f651e5 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -15,6 +15,44 @@ using namespace mfem; using namespace std; +class HyperelasticOperator : public TimeDependentOperator +{ + +protected: + ParBilinearForm *M, *S; + + CGSolver M_solver; // Krylov solver for inverting the mass matrix M + HypreSmoother M_prec; // Preconditioner for the mass matrix M + +public: + HyperelasticOperator(ParFiniteElementSpace &f, Array &ess_tdof_list_, + double visc, double mu, double K); + + /// Compute the right-hand side of the ODE system. + virtual void Mult(const Vector &vx, Vector &dvx_dt) const; + + double ElasticEnergy(const ParGridFunction &x) const; + double KineticEnergy(const ParGridFunction &v) const; + void GetElasticEnergyDensity(const ParGridFunction &x, + ParGridFunction &w) const; + + mutable Vector H_sp; + mutable Vector dvxdt_sp; + + ParFiniteElementSpace &fespace; + double viscosity; + Array ess_tdof_list; + ParNonlinearForm *H; + HyperelasticModel *model; + mutable Vector z; // auxiliary vector + mutable Vector z2; // auxiliary vector + HypreParMatrix *Mmat; // Mass matrix from ParallelAssemble() + HypreParMatrix Smat; + + virtual ~HyperelasticOperator(); +}; + + // Compute coefficients of the reduced integrator with respect to inputs Q and x // in HyperelasticNLFIntegrator_ComputeReducedEQP. void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, @@ -356,7 +394,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &h, Vector const &v, - NeoHookeanModel *model, Vector elfun, + HyperelasticOperator oper, Vector elfun, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); @@ -380,6 +418,13 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, DenseMatrix PMatO(dof, dim); Vector el_vect(dof * dim); + // Get nonlinear operator model + NonlinearForm *nl_H = oper.H; + //DomainNonlinearForm *dnf = nl_H->GetDNF(); + NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; + NeoHookeanModel *model = dynamic_cast(dnf); + + // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) { @@ -417,6 +462,8 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, } } + + // Compute action of nonlinear operator CalcInverse(Trans.Jacobian(), Jrt); fe.CalcDShape(ip, DSh); @@ -429,10 +476,27 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, // Calculate r[i] = v_i^T * el_vect // Perform the vector-matrix-vector multiplication: a^T * B * c - r[i] += v_i * el_vect; + r[i] += v_i * el_vect; // Elvect is added to in every iteration... // Scale by element transformation r[i] *= t; } + + // Error compared to FOM operator + Vector el_vect_test(dof * dim); + double error = 0.0; + dnf->AssembleElementVector(fe, Trans, elfun, el_vect_test); + // For each integration point + for (int i = 0; i < ir->GetNPoints(); i++) + { + // Get integration point + const IntegrationPoint &ip = ir->IntPoint(i); + error += el_vect[i] * ip.weight - el_vect_test[i]; + } + + + cout << "Element vector error = " << error << endl; + + } void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, @@ -494,7 +558,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const int nsets, const CAROM::Matrix *BH, const CAROM::Matrix *BH_snapshots, const bool precondition, const double nnls_tol, - const int maxNNLSnnz, NeoHookeanModel *model, + const int maxNNLSnnz, HyperelasticOperator oper, CAROM::Vector &sol) { const int nqe = ir0->GetNPoints(); @@ -590,7 +654,9 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, ph_i.GetSubVector(vdofs, elfun); // Compute the row of G corresponding to element e, store in r - ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, model, elfun, fe, *eltrans, r); + // Create vector r_test which will be compared to the nonlinear hyperelastic operator + + ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, elfun, fe, *eltrans, r); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; From 5a15f1533a5790aac5e8db1162102ca4226cdb5c Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:10:19 +0200 Subject: [PATCH 021/120] Debugging. Vshape not implemented for this class... --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 577f651e5..31e369928 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -435,7 +435,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Trans.SetIntPoint(&ip); // Evaluate the element shape functions at the integration point - fe.CalcVShape(Trans, trial_vshape); + //fe.CalcVShape(Trans, trial_vshape); // Get the transformation weight double t = Trans.Weight(); @@ -457,7 +457,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, for (int k = 0; k < spaceDim; ++k) { // h_i[k] += s * h[dofj] * trial_vshape(j, k); - v_i[k] += s * v[dofj] * trial_vshape(j, k); + v_i[k] += s * v[dofj] // * trial_vshape(j, k); // PMatI[j, k] = s * h[dofj]; } } From 4f9ae86e8c42a1310306c71044b87f87370547c0 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:11:35 +0200 Subject: [PATCH 022/120] fixed bug --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 31e369928..5b0582d92 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -457,7 +457,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, for (int k = 0; k < spaceDim; ++k) { // h_i[k] += s * h[dofj] * trial_vshape(j, k); - v_i[k] += s * v[dofj] // * trial_vshape(j, k); + v_i[k] += s * v[dofj]; // * trial_vshape(j, k); // PMatI[j, k] = s * h[dofj]; } } From 24a9e4c9f2cfa25842bfea2d5df8c82ba876401b Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:30:17 +0200 Subject: [PATCH 023/120] Explicitly adding model to eqp loop --- examples/prom/nonlinear_elasticity_global_rom.cpp | 3 ++- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 59de61714..ab0327fb0 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -739,6 +739,7 @@ int main(int argc, char *argv[]) // 9. Initialize the hyperelastic operator, the GLVis visualization and print // the initial energies. HyperelasticOperator oper(fespace, ess_tdof_list, visc, mu, K); + NeoHookeanModel* model = new NeoHookeanModel(mu, K); HyperelasticOperator *soper = 0; // Fill dvdt and dxdt @@ -917,7 +918,7 @@ int main(int argc, char *argv[]) eqpSol = new CAROM::Vector(ir0->GetNPoints() * fespace.GetNE(), true); SetupEQP_snapshots(ir0, myid, &fespace, nsets, H_librom, GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "H"), - preconditionNNLS, tolNNLS, maxNNLSnnz, oper, *eqpSol); + preconditionNNLS, tolNNLS, maxNNLSnnz, oper, model, *eqpSol); if (writeSampleMesh) WriteMeshEQP(pmesh, myid, ir0->GetNPoints(), *eqpSol); diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 5b0582d92..5f4b668fc 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -394,7 +394,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &h, Vector const &v, - HyperelasticOperator oper, Vector elfun, + HyperelasticOperator oper, NeoHookeanModel *model, Vector elfun, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); @@ -422,8 +422,6 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, NonlinearForm *nl_H = oper.H; //DomainNonlinearForm *dnf = nl_H->GetDNF(); NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; - NeoHookeanModel *model = dynamic_cast(dnf); - // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) @@ -469,7 +467,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, fe.CalcDShape(ip, DSh); Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); + model->EvalP(Jpt, P); //Problem with dynamic casting... AddMultABt(DS, P, PMatO); r[i] = 0.0; @@ -558,7 +556,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const int nsets, const CAROM::Matrix *BH, const CAROM::Matrix *BH_snapshots, const bool precondition, const double nnls_tol, - const int maxNNLSnnz, HyperelasticOperator oper, + const int maxNNLSnnz, HyperelasticOperator oper, NeoHookeanModel *model, CAROM::Vector &sol) { const int nqe = ir0->GetNPoints(); @@ -656,7 +654,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, // Compute the row of G corresponding to element e, store in r // Create vector r_test which will be compared to the nonlinear hyperelastic operator - ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, elfun, fe, *eltrans, r); + ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, model, elfun, fe, *eltrans, r); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; From e0c2c320e45f0eac1b283b359fb3b375dcff8041 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:41:53 +0200 Subject: [PATCH 024/120] segfault fix --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 5f4b668fc..0ed2673ec 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -419,9 +419,10 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector el_vect(dof * dim); // Get nonlinear operator model - NonlinearForm *nl_H = oper.H; + //NonlinearForm *nl_H = oper.H; //DomainNonlinearForm *dnf = nl_H->GetDNF(); - NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; + //NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; + NonlinearFormIntegrator* dnf = (*oper.H->GetDNFI())[0]; // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) @@ -467,7 +468,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, fe.CalcDShape(ip, DSh); Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); //Problem with dynamic casting... + model->EvalP(Jpt, P); AddMultABt(DS, P, PMatO); r[i] = 0.0; From fefcfd4cae6dfe10ca1881438aa3bc957c4f9b90 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:50:49 +0200 Subject: [PATCH 025/120] segfault fix 2 --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 0ed2673ec..3872322b2 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -419,10 +419,10 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector el_vect(dof * dim); // Get nonlinear operator model - //NonlinearForm *nl_H = oper.H; + NonlinearForm *nl_H = oper.H; //DomainNonlinearForm *dnf = nl_H->GetDNF(); //NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; - NonlinearFormIntegrator* dnf = (*oper.H->GetDNFI())[0]; + NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) From d78ae89f4bda1f5a361a462703cd01316d66a823 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:54:48 +0200 Subject: [PATCH 026/120] debug segfault --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 016626da8..2b3fdde15 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,8 @@ docs/html # Do not track results regression_tests/results +init_bash.sh +.gitignore +.vscode/settings.json +.vscode/libROM.code-workspace +init_bash_debug_arm.sh From 15ac7f04e06f07ee34165af96ab877a13a62ede0 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:57:22 +0200 Subject: [PATCH 027/120] actual debug fix --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 3872322b2..4e90e8c32 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -422,7 +422,8 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, NonlinearForm *nl_H = oper.H; //DomainNonlinearForm *dnf = nl_H->GetDNF(); //NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; - NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; + Array* dnfis = nl_H->GetDNFI(); + NonlinearFormIntegrator* dnf = (*(dnfis))[0]; // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) From a261c1d3cf649355f37fb82b07a4f0b2c5e6d11b Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 23:41:20 +0200 Subject: [PATCH 028/120] perhaps segfault fix --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 4e90e8c32..93ea028e5 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -419,7 +419,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector el_vect(dof * dim); // Get nonlinear operator model - NonlinearForm *nl_H = oper.H; + NonlinearForm nl_H = *oper.H; //DomainNonlinearForm *dnf = nl_H->GetDNF(); //NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; Array* dnfis = nl_H->GetDNFI(); @@ -656,11 +656,13 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, // Compute the row of G corresponding to element e, store in r // Create vector r_test which will be compared to the nonlinear hyperelastic operator - ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, model, elfun, fe, *eltrans, r); + ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, model, elfun, fe, *eltrans, r, el_vect); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; } + + } if (precondition) From a37321673456e3488885ca6f539c0715d58dd987 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 23:43:00 +0200 Subject: [PATCH 029/120] small fix --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 93ea028e5..4115dfcca 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -422,7 +422,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, NonlinearForm nl_H = *oper.H; //DomainNonlinearForm *dnf = nl_H->GetDNF(); //NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; - Array* dnfis = nl_H->GetDNFI(); + Array* dnfis = nl_H.GetDNFI(); NonlinearFormIntegrator* dnf = (*(dnfis))[0]; // For each integration point @@ -656,7 +656,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, // Compute the row of G corresponding to element e, store in r // Create vector r_test which will be compared to the nonlinear hyperelastic operator - ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, model, elfun, fe, *eltrans, r, el_vect); + ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, model, elfun, fe, *eltrans, r); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; From 5a2c71a48851ecbe2cbc1bb9d75fc843d07137c8 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 23:45:29 +0200 Subject: [PATCH 030/120] New test. --- .../nonlinear_elasticity_global_rom_eqp.hpp | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 4115dfcca..867c179d3 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -52,7 +52,6 @@ class HyperelasticOperator : public TimeDependentOperator virtual ~HyperelasticOperator(); }; - // Compute coefficients of the reduced integrator with respect to inputs Q and x // in HyperelasticNLFIntegrator_ComputeReducedEQP. void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, @@ -419,11 +418,12 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector el_vect(dof * dim); // Get nonlinear operator model - NonlinearForm nl_H = *oper.H; - //DomainNonlinearForm *dnf = nl_H->GetDNF(); - //NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; - Array* dnfis = nl_H.GetDNFI(); - NonlinearFormIntegrator* dnf = (*(dnfis))[0]; + NonlinearForm nl_H; + nl_H = *oper.H; + // DomainNonlinearForm *dnf = nl_H->GetDNF(); + // NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; + Array *dnfis = nl_H.GetDNFI(); + NonlinearFormIntegrator *dnf = (*(dnfis))[0]; // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) @@ -435,7 +435,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Trans.SetIntPoint(&ip); // Evaluate the element shape functions at the integration point - //fe.CalcVShape(Trans, trial_vshape); + // fe.CalcVShape(Trans, trial_vshape); // Get the transformation weight double t = Trans.Weight(); @@ -462,14 +462,12 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, } } - - // Compute action of nonlinear operator CalcInverse(Trans.Jacobian(), Jrt); fe.CalcDShape(ip, DSh); Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); + model->EvalP(Jpt, P); AddMultABt(DS, P, PMatO); r[i] = 0.0; @@ -493,10 +491,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, error += el_vect[i] * ip.weight - el_vect_test[i]; } - - cout << "Element vector error = " << error << endl; - - + cout << "Element vector error = " << error << endl; } void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, @@ -598,7 +593,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } // Get prolongation matrix - const Operator* P = fespace_H->GetProlongationMatrix(); + const Operator *P = fespace_H->GetProlongationMatrix(); // For every snapshot for (int i = 0; i < nsnap; ++i) @@ -655,14 +650,12 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, // Compute the row of G corresponding to element e, store in r // Create vector r_test which will be compared to the nonlinear hyperelastic operator - + ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, model, elfun, fe, *eltrans, r); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; } - - } if (precondition) From 79cd7aa9cca8bb932995442a5cc6f5c071f5cd04 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 23:49:28 +0200 Subject: [PATCH 031/120] testing --- .../nonlinear_elasticity_global_rom_eqp.hpp | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 867c179d3..4115dfcca 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -52,6 +52,7 @@ class HyperelasticOperator : public TimeDependentOperator virtual ~HyperelasticOperator(); }; + // Compute coefficients of the reduced integrator with respect to inputs Q and x // in HyperelasticNLFIntegrator_ComputeReducedEQP. void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, @@ -418,12 +419,11 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector el_vect(dof * dim); // Get nonlinear operator model - NonlinearForm nl_H; - nl_H = *oper.H; - // DomainNonlinearForm *dnf = nl_H->GetDNF(); - // NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; - Array *dnfis = nl_H.GetDNFI(); - NonlinearFormIntegrator *dnf = (*(dnfis))[0]; + NonlinearForm nl_H = *oper.H; + //DomainNonlinearForm *dnf = nl_H->GetDNF(); + //NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; + Array* dnfis = nl_H.GetDNFI(); + NonlinearFormIntegrator* dnf = (*(dnfis))[0]; // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) @@ -435,7 +435,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Trans.SetIntPoint(&ip); // Evaluate the element shape functions at the integration point - // fe.CalcVShape(Trans, trial_vshape); + //fe.CalcVShape(Trans, trial_vshape); // Get the transformation weight double t = Trans.Weight(); @@ -462,12 +462,14 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, } } + + // Compute action of nonlinear operator CalcInverse(Trans.Jacobian(), Jrt); fe.CalcDShape(ip, DSh); Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); + model->EvalP(Jpt, P); AddMultABt(DS, P, PMatO); r[i] = 0.0; @@ -491,7 +493,10 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, error += el_vect[i] * ip.weight - el_vect_test[i]; } - cout << "Element vector error = " << error << endl; + + cout << "Element vector error = " << error << endl; + + } void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, @@ -593,7 +598,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } // Get prolongation matrix - const Operator *P = fespace_H->GetProlongationMatrix(); + const Operator* P = fespace_H->GetProlongationMatrix(); // For every snapshot for (int i = 0; i < nsnap; ++i) @@ -650,12 +655,14 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, // Compute the row of G corresponding to element e, store in r // Create vector r_test which will be compared to the nonlinear hyperelastic operator - + ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, model, elfun, fe, *eltrans, r); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; } + + } if (precondition) From b9391634c3695101c169f281e2a009635498dcec Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 23:51:18 +0200 Subject: [PATCH 032/120] changed back --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 4115dfcca..53ed49b38 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -419,11 +419,11 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector el_vect(dof * dim); // Get nonlinear operator model - NonlinearForm nl_H = *oper.H; + NonlinearForm *nl_H = oper.H; //DomainNonlinearForm *dnf = nl_H->GetDNF(); //NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; - Array* dnfis = nl_H.GetDNFI(); - NonlinearFormIntegrator* dnf = (*(dnfis))[0]; + Array* dnfis = nl_H->GetDNFI(); + NonlinearFormIntegrator* dnf = (*(dnfis)[0]; // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) @@ -661,8 +661,6 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; } - - } if (precondition) From 0f86fe8c887e17b2f4b128dc0e782708cead0e32 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 12 Jun 2023 23:52:33 +0200 Subject: [PATCH 033/120] testing back again --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 53ed49b38..4e90e8c32 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -423,7 +423,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, //DomainNonlinearForm *dnf = nl_H->GetDNF(); //NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; Array* dnfis = nl_H->GetDNFI(); - NonlinearFormIntegrator* dnf = (*(dnfis)[0]; + NonlinearFormIntegrator* dnf = (*(dnfis))[0]; // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) From 8468f290d5c07b6ce6683d66a245b7df38f56b61 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 13 Jun 2023 00:10:56 +0200 Subject: [PATCH 034/120] Hopefully --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 4e90e8c32..7d3176a29 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -419,7 +419,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector el_vect(dof * dim); // Get nonlinear operator model - NonlinearForm *nl_H = oper.H; + ParNonlinearForm *nl_H = oper.H; //DomainNonlinearForm *dnf = nl_H->GetDNF(); //NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; Array* dnfis = nl_H->GetDNFI(); From 77fa0010fefc663bf396b39ce7a4c9275020a620 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 13 Jun 2023 01:16:08 +0200 Subject: [PATCH 035/120] Finished writing the test. --- .../nonlinear_elasticity_global_rom_eqp.hpp | 123 +++++++++++++----- 1 file changed, 87 insertions(+), 36 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 7d3176a29..607fcc6d7 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -52,7 +52,6 @@ class HyperelasticOperator : public TimeDependentOperator virtual ~HyperelasticOperator(); }; - // Compute coefficients of the reduced integrator with respect to inputs Q and x // in HyperelasticNLFIntegrator_ComputeReducedEQP. void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, @@ -394,7 +393,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &h, Vector const &v, - HyperelasticOperator oper, NeoHookeanModel *model, Vector elfun, + HyperelasticOperator oper, NeoHookeanModel *model, Vector elfun, Vector elvect, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); @@ -415,15 +414,16 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, DenseMatrix P(dim); DenseMatrix PMatI; // Extract element dofs PMatI.UseExternalData(elfun.GetData(), dof, dim); - DenseMatrix PMatO(dof, dim); - Vector el_vect(dof * dim); + DenseMatrix PMatO; + elvect.SetSize(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); // Get nonlinear operator model - ParNonlinearForm *nl_H = oper.H; - //DomainNonlinearForm *dnf = nl_H->GetDNF(); - //NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; - Array* dnfis = nl_H->GetDNFI(); - NonlinearFormIntegrator* dnf = (*(dnfis))[0]; + // ParNonlinearForm *nl_H = oper.H; + // DomainNonlinearForm *dnf = nl_H->GetDNF(); + // NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; + // Array* dnfis = nl_H->GetDNFI(); + // NonlinearFormIntegrator* dnf = (*(dnfis))[0]; // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) @@ -435,7 +435,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Trans.SetIntPoint(&ip); // Evaluate the element shape functions at the integration point - //fe.CalcVShape(Trans, trial_vshape); + // fe.CalcVShape(Trans, trial_vshape); // Get the transformation weight double t = Trans.Weight(); @@ -457,46 +457,40 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, for (int k = 0; k < spaceDim; ++k) { // h_i[k] += s * h[dofj] * trial_vshape(j, k); - v_i[k] += s * v[dofj]; // * trial_vshape(j, k); + v_i[k] += s * v[dofj]; // * trial_vshape(j, k); // TODO why no v_shape? // PMatI[j, k] = s * h[dofj]; } } - - // Compute action of nonlinear operator CalcInverse(Trans.Jacobian(), Jrt); fe.CalcDShape(ip, DSh); Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); + model->EvalP(Jpt, P); + P *= ip.weight * t; // TEST Scale by integration point weight AddMultABt(DS, P, PMatO); r[i] = 0.0; - // Calculate r[i] = v_i^T * el_vect + // Calculate r[i] = v_i^T * elvect // Perform the vector-matrix-vector multiplication: a^T * B * c - r[i] += v_i * el_vect; // Elvect is added to in every iteration... - // Scale by element transformation - r[i] *= t; + r[i] += v_i * elvect; // Elvect is added to in every iteration... } // Error compared to FOM operator - Vector el_vect_test(dof * dim); - double error = 0.0; - dnf->AssembleElementVector(fe, Trans, elfun, el_vect_test); + // Vector el_vect_test(dof * dim); + // double error = 0.0; + // dnf->AssembleElementVector(fe, Trans, elfun, el_vect_test); // For each integration point - for (int i = 0; i < ir->GetNPoints(); i++) - { - // Get integration point - const IntegrationPoint &ip = ir->IntPoint(i); - error += el_vect[i] * ip.weight - el_vect_test[i]; - } - - - cout << "Element vector error = " << error << endl; - - + /* for (int i = 0; i < ir->GetNPoints(); i++) + { + // Get integration point + const IntegrationPoint &ip = ir->IntPoint(i); + error += elvect[i] * ip.weight - el_vect_test[i]; + } */ + + // cout << "Element vector error = " << error << endl; } void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, @@ -598,7 +592,12 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } // Get prolongation matrix - const Operator* P = fespace_H->GetProlongationMatrix(); + Vector aux1, aux2; + const Operator *P = fespace_H->GetProlongationMatrix(); + if (P) + { + aux2.SetSize(P->Height()); + } // For every snapshot for (int i = 0; i < nsnap; ++i) @@ -607,6 +606,15 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, for (int j = 0; j < BH_snapshots->numRows(); ++j) h_i[j] = (*BH_snapshots)(j, i); + // TEST initialize y and py + Vector y(h_i); + Vector y_true(h_i); + Vector &py = P ? aux2 : y; + py = 0.0; + + // TEST initialize element vector + Vector elvect; + // Get prolongated dofs // See: https://docs.mfem.org/html/nonlinearform_8cpp_source.html#l00131 Vector ph_i; @@ -652,15 +660,58 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const FiniteElement &fe = *fespace_H->GetFE(e); ElementTransformation *eltrans = fespace_H->GetElementTransformation(e); ph_i.GetSubVector(vdofs, elfun); - + if (doftrans) + { + doftrans->InvTransformPrimal(elfun); + } // Compute the row of G corresponding to element e, store in r // Create vector r_test which will be compared to the nonlinear hyperelastic operator - - ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, model, elfun, fe, *eltrans, r); + + ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, model, elfun, elvect, fe, *eltrans, r); + + if (doftrans) + { + doftrans->TransformDual(elvect); + } + if (e == 0) + { + py.AddElementVector(vdofs, elvect); + } for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; } + /* if (Serial()) + { + MFEM_ABORT("öh no!") + if (cP) + { + cP->MultTranspose(py, y); + } + + for (int i = 0; i < ess_tdof_list.Size(); i++) + { + y(ess_tdof_list[i]) = 0.0; + } + // y(ess_tdof_list[i]) = x(ess_tdof_list[i]); + }*/ + + P->MultTranspose(aux2, y); + + const int N = oper.ess_tdof_list.Size(); + const auto idx = oper.ess_tdof_list.Read(); + auto Y_RW = y.ReadWrite(); + MFEM_FORALL(i, N, Y_RW[idx[i]] = 0.0; ); + double error = 0.0; + + oper.H->Mult(h_i, y_true); + + for (int ii = 0; ii < y.Size(); ii++) + { + error += abs(y[ii] - y_true[ii]); + } + + cout << "Element vector error = " << error << endl; } if (precondition) From 32d3338194a7cabfbc9319894e0a2712034bd376 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 13 Jun 2023 03:16:04 +0200 Subject: [PATCH 036/120] Fixed compilation --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 607fcc6d7..286e50640 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -700,8 +700,10 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const int N = oper.ess_tdof_list.Size(); const auto idx = oper.ess_tdof_list.Read(); - auto Y_RW = y.ReadWrite(); - MFEM_FORALL(i, N, Y_RW[idx[i]] = 0.0; ); + // auto Y_RW = y.ReadWrite(); + // MFEM_FORALL(i, N, Y_RW[idx[i]] = 0.0; ); + for (int ii = 0; ii < N; ++ii) + y[idx[ii]] = 0.0; double error = 0.0; oper.H->Mult(h_i, y_true); From e7cdfe29dc228e11530259dbfe1025d8cf2e050a Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 13 Jun 2023 03:33:09 +0200 Subject: [PATCH 037/120] Debugging elvect size --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 286e50640..a02c61d39 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -393,7 +393,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &h, Vector const &v, - HyperelasticOperator oper, NeoHookeanModel *model, Vector elfun, Vector elvect, + HyperelasticOperator const oper, NeoHookeanModel const *model, Vector const elfun, Vector elvect, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); @@ -415,9 +415,12 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, DenseMatrix PMatI; // Extract element dofs PMatI.UseExternalData(elfun.GetData(), dof, dim); DenseMatrix PMatO; + cout << "B, elvect size = " << elvect.Size() << endl; elvect.SetSize(dof * dim); PMatO.UseExternalData(elvect.GetData(), dof, dim); + cout << "A, elvect size = " << elvect.Size() << endl; + // Get nonlinear operator model // ParNonlinearForm *nl_H = oper.H; // DomainNonlinearForm *dnf = nl_H->GetDNF(); @@ -478,6 +481,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, r[i] += v_i * elvect; // Elvect is added to in every iteration... } + cout << "C, elvect size = " << elvect.Size() << endl; // Error compared to FOM operator // Vector el_vect_test(dof * dim); // double error = 0.0; @@ -671,10 +675,12 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, if (doftrans) { + cout << "doftrans is true " << endl; doftrans->TransformDual(elvect); } if (e == 0) { + cout << "D, elvect size = " << elvect.Size() << endl; py.AddElementVector(vdofs, elvect); } From 5cfd5a3057b16a3cb7e8f85cd2f0df2a260b32bf Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 13 Jun 2023 03:40:41 +0200 Subject: [PATCH 038/120] passing by reference --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index a02c61d39..80c72bb1c 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -393,7 +393,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &h, Vector const &v, - HyperelasticOperator const oper, NeoHookeanModel const *model, Vector const elfun, Vector elvect, + HyperelasticOperator const &oper, NeoHookeanModel const *model, Vector const &elfun, Vector &elvect, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); @@ -415,11 +415,11 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, DenseMatrix PMatI; // Extract element dofs PMatI.UseExternalData(elfun.GetData(), dof, dim); DenseMatrix PMatO; - cout << "B, elvect size = " << elvect.Size() << endl; + cout << "A, elvect size = " << elvect.Size() << endl; elvect.SetSize(dof * dim); PMatO.UseExternalData(elvect.GetData(), dof, dim); - cout << "A, elvect size = " << elvect.Size() << endl; + cout << "B, elvect size = " << elvect.Size() << endl; // Get nonlinear operator model // ParNonlinearForm *nl_H = oper.H; From aa00f1c25d1df43257a1853947f7e010b1881152 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 13 Jun 2023 03:47:41 +0200 Subject: [PATCH 039/120] Fix for element vector --- .../nonlinear_elasticity_global_rom_eqp.hpp | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 80c72bb1c..fc87249de 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -415,12 +415,9 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, DenseMatrix PMatI; // Extract element dofs PMatI.UseExternalData(elfun.GetData(), dof, dim); DenseMatrix PMatO; - cout << "A, elvect size = " << elvect.Size() << endl; elvect.SetSize(dof * dim); PMatO.UseExternalData(elvect.GetData(), dof, dim); - cout << "B, elvect size = " << elvect.Size() << endl; - // Get nonlinear operator model // ParNonlinearForm *nl_H = oper.H; // DomainNonlinearForm *dnf = nl_H->GetDNF(); @@ -481,7 +478,6 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, r[i] += v_i * elvect; // Elvect is added to in every iteration... } - cout << "C, elvect size = " << elvect.Size() << endl; // Error compared to FOM operator // Vector el_vect_test(dof * dim); // double error = 0.0; @@ -678,11 +674,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, cout << "doftrans is true " << endl; doftrans->TransformDual(elvect); } - if (e == 0) - { - cout << "D, elvect size = " << elvect.Size() << endl; - py.AddElementVector(vdofs, elvect); - } + + py.AddElementVector(vdofs, elvect); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; @@ -701,25 +694,27 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } // y(ess_tdof_list[i]) = x(ess_tdof_list[i]); }*/ + if (j== 1 && i == 1) + { + P->MultTranspose(aux2, y); - P->MultTranspose(aux2, y); - - const int N = oper.ess_tdof_list.Size(); - const auto idx = oper.ess_tdof_list.Read(); - // auto Y_RW = y.ReadWrite(); - // MFEM_FORALL(i, N, Y_RW[idx[i]] = 0.0; ); - for (int ii = 0; ii < N; ++ii) - y[idx[ii]] = 0.0; - double error = 0.0; + const int N = oper.ess_tdof_list.Size(); + const auto idx = oper.ess_tdof_list.Read(); + // auto Y_RW = y.ReadWrite(); + // MFEM_FORALL(i, N, Y_RW[idx[i]] = 0.0; ); + for (int ii = 0; ii < N; ++ii) + y[idx[ii]] = 0.0; + double error = 0.0; - oper.H->Mult(h_i, y_true); + oper.H->Mult(h_i, y_true); - for (int ii = 0; ii < y.Size(); ii++) - { - error += abs(y[ii] - y_true[ii]); - } + for (int ii = 0; ii < y.Size(); ii++) + { + error += abs(y[ii] - y_true[ii]); + } - cout << "Element vector error = " << error << endl; + cout << "Element vector error = " << error << endl; + } } if (precondition) From 0c25a027ab8cb2d144ef0cc2023edbc1cc27f627 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 13 Jun 2023 03:50:35 +0200 Subject: [PATCH 040/120] check all errors --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index fc87249de..ffeb427c0 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -709,7 +709,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, oper.H->Mult(h_i, y_true); for (int ii = 0; ii < y.Size(); ii++) - { + { cout << "y[ii] = " << y[ii] << "y_true[ii] = " << y_true[ii] << endl; error += abs(y[ii] - y_true[ii]); } From dabe450e072c82ccadc58e9b6b98fc87777b7961 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 13 Jun 2023 04:06:15 +0200 Subject: [PATCH 041/120] Synced with hyperelasticintegrator. --- examples/prom/nonlinear_elasticity_global_rom.cpp | 6 +++--- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index ab0327fb0..581d791b0 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -906,10 +906,10 @@ int main(int argc, char *argv[]) { // int order = 2 * el.GetOrder(); const FiniteElement &fe = *fespace.GetFE(0); - ElementTransformation *eltrans = fespace.GetElementTransformation(0); + //ElementTransformation *eltrans = fespace.GetElementTransformation(0); - int order = eltrans->OrderW() + 2 * fe.GetOrder(); - ir0 = &IntRules.Get(fe.GetGeomType(), order); + //int order = eltrans->OrderW() + 2 * fe.GetOrder(); + ir0 = &(IntRules.Get(fe.GetGeomType(), 2*fe.GetOrder() + 3)); } if (use_eqp) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index ffeb427c0..6f0d16a86 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -393,7 +393,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &h, Vector const &v, - HyperelasticOperator const &oper, NeoHookeanModel const *model, Vector const &elfun, Vector &elvect, + HyperelasticOperator const &oper, NeoHookeanModel *model, Vector const &elfun, Vector &elvect, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); @@ -425,6 +425,9 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, // Array* dnfis = nl_H->GetDNFI(); // NonlinearFormIntegrator* dnf = (*(dnfis))[0]; + elvect = 0.0; + model->SetTransformation(Trans); + // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) { @@ -709,7 +712,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, oper.H->Mult(h_i, y_true); for (int ii = 0; ii < y.Size(); ii++) - { cout << "y[ii] = " << y[ii] << "y_true[ii] = " << y_true[ii] << endl; + { cout << "y[ii] = " << y[ii] << ", y_true[ii] = " << y_true[ii] << endl; error += abs(y[ii] - y_true[ii]); } From c76f3a0644b903fe243d19a1ceac903788654f08 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 13 Jun 2023 04:10:23 +0200 Subject: [PATCH 042/120] Divide error by two --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 6f0d16a86..20c9badd9 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -712,8 +712,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, oper.H->Mult(h_i, y_true); for (int ii = 0; ii < y.Size(); ii++) - { cout << "y[ii] = " << y[ii] << ", y_true[ii] = " << y_true[ii] << endl; - error += abs(y[ii] - y_true[ii]); + { cout << "y[ii] = " << y[ii]/2 << ", y_true[ii] = " << y_true[ii] << endl; + error += abs(y[ii]/2 - y_true[ii]); } cout << "Element vector error = " << error << endl; From c14a422ccca68aea34144176bbb41f6a3cae80ac Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 13 Jun 2023 04:13:56 +0200 Subject: [PATCH 043/120] changed points to check error --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 20c9badd9..b2076666d 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -697,7 +697,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } // y(ess_tdof_list[i]) = x(ess_tdof_list[i]); }*/ - if (j== 1 && i == 1) + if (j== 0 && i == 0) { P->MultTranspose(aux2, y); @@ -712,8 +712,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, oper.H->Mult(h_i, y_true); for (int ii = 0; ii < y.Size(); ii++) - { cout << "y[ii] = " << y[ii]/2 << ", y_true[ii] = " << y_true[ii] << endl; - error += abs(y[ii]/2 - y_true[ii]); + { cout << "y[ii] = " << y[ii] << ", y_true[ii] = " << y_true[ii] << endl; + error += abs(y[ii] - y_true[ii]); } cout << "Element vector error = " << error << endl; From 28aacb5fa9015d605a0f05cfe3c3eeb1d086c7e4 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 13 Jun 2023 04:24:26 +0200 Subject: [PATCH 044/120] Clean up --- .../nonlinear_elasticity_global_rom_eqp.hpp | 131 ++++++------------ 1 file changed, 40 insertions(+), 91 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index b2076666d..325e8d374 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -418,13 +418,6 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, elvect.SetSize(dof * dim); PMatO.UseExternalData(elvect.GetData(), dof, dim); - // Get nonlinear operator model - // ParNonlinearForm *nl_H = oper.H; - // DomainNonlinearForm *dnf = nl_H->GetDNF(); - // NonlinearFormIntegrator* dnf = (*(nl_H->GetDNFI()))[0]; - // Array* dnfis = nl_H->GetDNFI(); - // NonlinearFormIntegrator* dnf = (*(dnfis))[0]; - elvect = 0.0; model->SetTransformation(Trans); @@ -438,7 +431,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Trans.SetIntPoint(&ip); // Evaluate the element shape functions at the integration point - // fe.CalcVShape(Trans, trial_vshape); + // fe.CalcVShape(Trans, trial_vshape); // Not implemented // Get the transformation weight double t = Trans.Weight(); @@ -447,53 +440,34 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, h_i = 0.0; v_i = 0.0; + // Compute action of nonlinear operator + CalcInverse(Trans.Jacobian(), Jrt); + fe.CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P); + P *= ip.weight * t; // TEST Scale by integration point weight + AddMultABt(DS, P, PMatO); + + r[i] = 0.0; + // For every dof in element for (int j = 0; j < dof; ++j) { - // Q: Why can vdofs[j] < 0 be true? const int dofj = (vdofs[j] >= 0) ? vdofs[j] : -1 - vdofs[j]; const double s = (vdofs[j] >= 0) ? 1.0 : -1.0; - // Calculate h_i = B_e * h_e, where h_e are the samples - // for this snapshot at this element dofs - // Also calculate v_i = B_e * v_e + // Also Calculate v_i = B_e * v_e for (int k = 0; k < spaceDim; ++k) { - // h_i[k] += s * h[dofj] * trial_vshape(j, k); v_i[k] += s * v[dofj]; // * trial_vshape(j, k); // TODO why no v_shape? - // PMatI[j, k] = s * h[dofj]; } } - // Compute action of nonlinear operator - CalcInverse(Trans.Jacobian(), Jrt); - fe.CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); - P *= ip.weight * t; // TEST Scale by integration point weight - AddMultABt(DS, P, PMatO); - - r[i] = 0.0; - // Calculate r[i] = v_i^T * elvect // Perform the vector-matrix-vector multiplication: a^T * B * c r[i] += v_i * elvect; // Elvect is added to in every iteration... } - - // Error compared to FOM operator - // Vector el_vect_test(dof * dim); - // double error = 0.0; - // dnf->AssembleElementVector(fe, Trans, elfun, el_vect_test); - // For each integration point - /* for (int i = 0; i < ir->GetNPoints(); i++) - { - // Get integration point - const IntegrationPoint &ip = ir->IntPoint(i); - error += elvect[i] * ip.weight - el_vect_test[i]; - } */ - - // cout << "Element vector error = " << error << endl; } void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, @@ -601,6 +575,10 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, { aux2.SetSize(P->Height()); } + else + { + MFEM_ABORT("P is null, generalize to serial case") + } // For every snapshot for (int i = 0; i < nsnap; ++i) @@ -612,29 +590,18 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, // TEST initialize y and py Vector y(h_i); Vector y_true(h_i); - Vector &py = P ? aux2 : y; - py = 0.0; + Vector &py = aux2; // TEST initialize element vector Vector elvect; // Get prolongated dofs - // See: https://docs.mfem.org/html/nonlinearform_8cpp_source.html#l00131 Vector ph_i; Vector elfun; - if (P) - { - ph_i.SetSize(P->Height()); - P->Mult(h_i, ph_i); - } - else - { - ph_i.SetSize(h_i.Size()); - for (int j = 0; j < h_i.Size(); ++j) - ph_i[j] = h_i[j]; - } - // Q: Not sure what this does + ph_i.SetSize(P->Height()); + P->Mult(h_i, ph_i); + if (skipFirstW && i > 0 && i % nsnapPerSet == 0) skip++; @@ -645,6 +612,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, // For each basis vector for (int j = 0; j < NB; ++j) { + py = 0.0; + // Get basis vector for (int k = 0; k < BH->numRows(); ++k) v_j[k] = (*BH)(k, j); @@ -665,59 +634,39 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, ph_i.GetSubVector(vdofs, elfun); if (doftrans) { - doftrans->InvTransformPrimal(elfun); + MFEM_ABORT("Doftrans is true, make corresponding edits") } // Compute the row of G corresponding to element e, store in r // Create vector r_test which will be compared to the nonlinear hyperelastic operator ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, model, elfun, elvect, fe, *eltrans, r); - if (doftrans) - { - cout << "doftrans is true " << endl; - doftrans->TransformDual(elvect); - } - py.AddElementVector(vdofs, elvect); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; } - /* if (Serial()) - { - MFEM_ABORT("öh no!") - if (cP) - { - cP->MultTranspose(py, y); - } - - for (int i = 0; i < ess_tdof_list.Size(); i++) - { - y(ess_tdof_list[i]) = 0.0; - } - // y(ess_tdof_list[i]) = x(ess_tdof_list[i]); - }*/ - if (j== 0 && i == 0) - { - P->MultTranspose(aux2, y); - const int N = oper.ess_tdof_list.Size(); - const auto idx = oper.ess_tdof_list.Read(); - // auto Y_RW = y.ReadWrite(); - // MFEM_FORALL(i, N, Y_RW[idx[i]] = 0.0; ); - for (int ii = 0; ii < N; ++ii) - y[idx[ii]] = 0.0; - double error = 0.0; + if (j == 0 && i == 0) + { + P->MultTranspose(aux2, y); - oper.H->Mult(h_i, y_true); + const int N = oper.ess_tdof_list.Size(); + const auto idx = oper.ess_tdof_list.Read(); + for (int ii = 0; ii < N; ++ii) + y[idx[ii]] = 0.0; + double error = 0.0; - for (int ii = 0; ii < y.Size(); ii++) - { cout << "y[ii] = " << y[ii] << ", y_true[ii] = " << y_true[ii] << endl; - error += abs(y[ii] - y_true[ii]); - } + oper.H->Mult(h_i, y_true); - cout << "Element vector error = " << error << endl; + for (int ii = 0; ii < y.Size(); ii++) + { + cout << "y[ii] = " << y[ii] << ", y_true[ii] = " << y_true[ii] << endl; + error += abs(y[ii] - y_true[ii]); } + + cout << "Element vector error = " << error << endl; + } } if (precondition) From 40564fd45f96b38c9d7707603d0051e224d28dd3 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 14 Jun 2023 00:18:49 +0200 Subject: [PATCH 045/120] testing SetupEQPSnapshots --- .../nonlinear_elasticity_global_rom_eqp.hpp | 101 ++++-------------- 1 file changed, 19 insertions(+), 82 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 325e8d374..6cf2f79d1 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -392,21 +392,14 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes */ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, - Vector const &h, Vector const &v, - HyperelasticOperator const &oper, NeoHookeanModel *model, Vector const &elfun, Vector &elvect, + Vector const &ve_j, NeoHookeanModel *model, Vector const &elfun, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); int dof = fe.GetDof(); // Get number of dofs in element - int spaceDim = Trans.GetSpaceDim(); - - Vector h_i(spaceDim); - Vector v_i(spaceDim); - - DenseMatrix trial_vshape(dof, spaceDim); + int dim = fe.GetDim(); // Initialize nonlinear operator matrices (there is probably a better way) - int dim = fe.GetDim(); DenseMatrix DSh(dof, dim); DenseMatrix DS(dof, dim); DenseMatrix Jrt(dim); @@ -415,58 +408,40 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, DenseMatrix PMatI; // Extract element dofs PMatI.UseExternalData(elfun.GetData(), dof, dim); DenseMatrix PMatO; - elvect.SetSize(dof * dim); + Vector elvect(dof * dim); PMatO.UseExternalData(elvect.GetData(), dof, dim); - elvect = 0.0; model->SetTransformation(Trans); // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) { + elvect = 0.0; // Get integration point const IntegrationPoint &ip = ir->IntPoint(i); // Set integration point in the element transformation Trans.SetIntPoint(&ip); - // Evaluate the element shape functions at the integration point - // fe.CalcVShape(Trans, trial_vshape); // Not implemented - // Get the transformation weight double t = Trans.Weight(); - // Initialize h_i and v_i - h_i = 0.0; - v_i = 0.0; - // Compute action of nonlinear operator CalcInverse(Trans.Jacobian(), Jrt); fe.CalcDShape(ip, DSh); Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); model->EvalP(Jpt, P); - P *= ip.weight * t; // TEST Scale by integration point weight + P *= t; // NB: Not by ip.weight AddMultABt(DS, P, PMatO); r[i] = 0.0; - // For every dof in element - for (int j = 0; j < dof; ++j) + // Calculate r[i] = ve_j^T * elvect + for (int k = 0; k < elvect.Size(); k++) { - const int dofj = (vdofs[j] >= 0) ? vdofs[j] : -1 - vdofs[j]; - const double s = (vdofs[j] >= 0) ? 1.0 : -1.0; - - // Also Calculate v_i = B_e * v_e - for (int k = 0; k < spaceDim; ++k) - { - v_i[k] += s * v[dofj]; // * trial_vshape(j, k); // TODO why no v_shape? - } + r[i] += ve_j[k] * elvect[k]; } - - // Calculate r[i] = v_i^T * elvect - // Perform the vector-matrix-vector multiplication: a^T * B * c - r[i] += v_i * elvect; // Elvect is added to in every iteration... } } @@ -569,13 +544,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } // Get prolongation matrix - Vector aux1, aux2; const Operator *P = fespace_H->GetProlongationMatrix(); - if (P) - { - aux2.SetSize(P->Height()); - } - else + if (!P) { MFEM_ABORT("P is null, generalize to serial case") } @@ -587,14 +557,6 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, for (int j = 0; j < BH_snapshots->numRows(); ++j) h_i[j] = (*BH_snapshots)(j, i); - // TEST initialize y and py - Vector y(h_i); - Vector y_true(h_i); - Vector &py = aux2; - - // TEST initialize element vector - Vector elvect; - // Get prolongated dofs Vector ph_i; Vector elfun; @@ -605,22 +567,20 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, if (skipFirstW && i > 0 && i % nsnapPerSet == 0) skip++; - // Set grid function for h_i - ParGridFunction hi_gf(fespace_H); - hi_gf.SetFromTrueDofs(h_i); - // For each basis vector for (int j = 0; j < NB; ++j) { - py = 0.0; - + // Get basis vector for (int k = 0; k < BH->numRows(); ++k) v_j[k] = (*BH)(k, j); - // Set grid function for basis vector - ParGridFunction vj_gf(fespace_H); - vj_gf.SetFromTrueDofs(v_j); + // Get prolongated dofs + Vector pv_j; + Vector ve_j; + + pv_j.SetSize(P->Height()); + P->Mult(v_j, pv_j); // TODO: is it better to make the element loop the outer loop? // For each element @@ -632,41 +592,18 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const FiniteElement &fe = *fespace_H->GetFE(e); ElementTransformation *eltrans = fespace_H->GetElementTransformation(e); ph_i.GetSubVector(vdofs, elfun); + pv_j.GetSubVector(vdofs, ve_j); if (doftrans) { MFEM_ABORT("Doftrans is true, make corresponding edits") } - // Compute the row of G corresponding to element e, store in r - // Create vector r_test which will be compared to the nonlinear hyperelastic operator - ComputeElementRowOfG(ir0, vdofs, hi_gf, vj_gf, oper, model, elfun, elvect, fe, *eltrans, r); - - py.AddElementVector(vdofs, elvect); + // Compute the row of G corresponding to element e, store in r + ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; } - - if (j == 0 && i == 0) - { - P->MultTranspose(aux2, y); - - const int N = oper.ess_tdof_list.Size(); - const auto idx = oper.ess_tdof_list.Read(); - for (int ii = 0; ii < N; ++ii) - y[idx[ii]] = 0.0; - double error = 0.0; - - oper.H->Mult(h_i, y_true); - - for (int ii = 0; ii < y.Size(); ii++) - { - cout << "y[ii] = " << y[ii] << ", y_true[ii] = " << y_true[ii] << endl; - error += abs(y[ii] - y_true[ii]); - } - - cout << "Element vector error = " << error << endl; - } } if (precondition) From eeb59421a54cffd57ebd7cfa8cba0791f5698379 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 14 Jun 2023 00:38:19 +0200 Subject: [PATCH 046/120] testing errors for NNLS --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 6cf2f79d1..ad5028dc2 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -442,6 +442,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, { r[i] += ve_j[k] * elvect[k]; } + cout << "r[i] = " << r[i]; } } @@ -622,6 +623,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, for (int j = 0; j < nqe; ++j) w((i * nqe) + j) = w_el[j]; } + bool stop = true; SolveNNLS(rank, nnls_tol, maxNNLSnnz, w, Gt, sol); } From 031d370f7165a869c322b8a80d3bdb25a2a46c1e Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 14 Jun 2023 00:41:55 +0200 Subject: [PATCH 047/120] small fix --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index ad5028dc2..fa1f5ba4d 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -442,7 +442,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, { r[i] += ve_j[k] * elvect[k]; } - cout << "r[i] = " << r[i]; + cout << "r[i] = " << r[i] << endl; } } From d2d0b797c3fff0c9faed04b0e8373d7152a5ff88 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 14 Jun 2023 01:12:52 +0200 Subject: [PATCH 048/120] removed testing --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index fa1f5ba4d..f5f12d5a2 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -442,7 +442,7 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, { r[i] += ve_j[k] * elvect[k]; } - cout << "r[i] = " << r[i] << endl; + } } From 7d4a9fa1bd2f1199cec8e57f3ec00fd3c120087f Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 19 Jun 2023 11:42:51 +0200 Subject: [PATCH 049/120] Remove normalize constraints --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index f5f12d5a2..93cdd5b52 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -467,7 +467,7 @@ void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, rhs_ub(i) += delta; } - nnls.normalize_constraints(Gt, rhs_lb, rhs_ub); + // nnls.normalize_constraints(Gt, rhs_lb, rhs_ub); nnls.solve_parallel_with_scalapack(Gt, rhs_lb, rhs_ub, sol); int nnz = 0; From a76649b2ab0cf55559ceec681345ffd67472faee Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 19 Jun 2023 16:19:42 +0200 Subject: [PATCH 050/120] Fixed segfault --- .../prom/nonlinear_elasticity_global_rom.cpp | 33 ++++++++----------- .../nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 581d791b0..f390d8146 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -200,7 +200,7 @@ class RomOperator : public TimeDependentOperator std::vector eqp_rw; std::vector eqp_qp; Vector eqp_coef; - const bool fastIntegration = false; //TODO: implement fast integration + const bool fastIntegration = false; // TODO: implement fast integration int rank; @@ -739,7 +739,7 @@ int main(int argc, char *argv[]) // 9. Initialize the hyperelastic operator, the GLVis visualization and print // the initial energies. HyperelasticOperator oper(fespace, ess_tdof_list, visc, mu, K); - NeoHookeanModel* model = new NeoHookeanModel(mu, K); + NeoHookeanModel *model = new NeoHookeanModel(mu, K); HyperelasticOperator *soper = 0; // Fill dvdt and dxdt @@ -904,12 +904,8 @@ int main(int argc, char *argv[]) if (ir0 == NULL) { - // int order = 2 * el.GetOrder(); const FiniteElement &fe = *fespace.GetFE(0); - //ElementTransformation *eltrans = fespace.GetElementTransformation(0); - - //int order = eltrans->OrderW() + 2 * fe.GetOrder(); - ir0 = &(IntRules.Get(fe.GetGeomType(), 2*fe.GetOrder() + 3)); + ir0 = &(IntRules.Get(fe.GetGeomType(), 2 * fe.GetOrder() + 3)); } if (use_eqp) @@ -918,7 +914,7 @@ int main(int argc, char *argv[]) eqpSol = new CAROM::Vector(ir0->GetNPoints() * fespace.GetNE(), true); SetupEQP_snapshots(ir0, myid, &fespace, nsets, H_librom, GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "H"), - preconditionNNLS, tolNNLS, maxNNLSnnz, oper, model, *eqpSol); + preconditionNNLS, tolNNLS, maxNNLSnnz, model, *eqpSol); if (writeSampleMesh) WriteMeshEQP(pmesh, myid, ir0->GetNPoints(), *eqpSol); @@ -1145,7 +1141,6 @@ int main(int argc, char *argv[]) } ode_solver->Init(*romop); - } else { @@ -1585,14 +1580,18 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, CAROM::Vector *eqpSol, const IntegrationRule *ir_eqp_) : TimeDependentOperator(rxdim_ + rvdim_, 0.0), fom(fom_), fomSp(fomSp_), rxdim(rxdim_), rvdim(rvdim_), hdim(hdim_), x0(x0_), v0(v0_), v0_fom(v0_fom_), - smm(smm_), nsamp_H(smm_->GetNumVarSamples("H")), V_x(*V_x_), V_v(*V_v_), - U_H(U_H_), Hsinv(Hsinv_), zN(std::max(nsamp_H, 1), false), zX(std::max(nsamp_H, - 1), - false), + smm(smm_), V_x(*V_x_), V_v(*V_v_), U_H(U_H_), Hsinv(Hsinv_), M_hat_solver(fom_->fespace.GetComm()), oversampling(oversampling_), z(height / 2), hyperreduce(hyperreduce_), x_base_only(x_base_only_), eqp(use_eqp), ir_eqp(ir_eqp_) { + if (!eqp) + { + nsamp_H = smm_->GetNumVarSamples("H"); + zN = CAROM::Vector(std::max(nsamp_H, 1), false); + zX = CAROM::Vector(std::max(nsamp_H, 1), false); + } + if (myid == 0 && !eqp) { V_v_sp = new CAROM::Matrix(fomSp->Height() / 2, rvdim, false); @@ -1613,7 +1612,6 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, smm->GatherDistributedMatrixRows("X", V_x, rxdim, *V_x_sp); } - // Create V_vTU_H, for hyperreduction V_v.transposeMult(*U_H, V_vTU_H); @@ -1627,13 +1625,11 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, // Create S_hat ComputeCtAB(fom->Smat, V_v, V_v, *S_hat); - // Apply S_hat to the initial velocity and store fom->Smat.Mult(v0_fom, *S_hat_v0_temp); V_v.transposeMult(*S_hat_v0_temp_librom, S_hat_v0); - // Create M_hat - ComputeCtAB(*fom->Mmat, V_v, V_v, *M_hat); + ComputeCtAB(*(fom->Mmat), V_v, V_v, *M_hat); // Invert M_hat and store M_hat->inverse(*M_hat_inv); @@ -1734,7 +1730,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const CAROM::Vector dx_dt_librom(dx_dt.GetData(), rxdim, false, false); if (eqp) - { // Lift v-vector and save + { // Lift v-vector and save V_v.mult(v_librom, *z_v_librom); V_x.transposeMult(*z_v_librom, dx_dt_librom); @@ -1809,7 +1805,6 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const z.Neg(); // z = -z M_hat_inv->mult(*z_librom, dv_dt_librom); // to invert reduced mass matrix operator. - } void RomOperator::Mult_FullOrder(const Vector &vx, Vector &dvx_dt) const diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 93cdd5b52..26bff7337 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -505,7 +505,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const int nsets, const CAROM::Matrix *BH, const CAROM::Matrix *BH_snapshots, const bool precondition, const double nnls_tol, - const int maxNNLSnnz, HyperelasticOperator oper, NeoHookeanModel *model, + const int maxNNLSnnz, NeoHookeanModel *model, CAROM::Vector &sol) { const int nqe = ir0->GetNPoints(); From d961afc49365e38e7e9525690731254e9e44adcd Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 20 Jun 2023 11:58:46 +0200 Subject: [PATCH 051/120] Reduced EQP is building --- .../prom/nonlinear_elasticity_global_rom.cpp | 16 +- .../nonlinear_elasticity_global_rom_eqp.hpp | 327 ++++-------------- 2 files changed, 77 insertions(+), 266 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index f390d8146..a1b0825f8 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -204,6 +204,8 @@ class RomOperator : public TimeDependentOperator int rank; + NeoHookeanModel *model; + protected: CAROM::Matrix *S_hat; CAROM::Vector *S_hat_v0; @@ -229,7 +231,7 @@ class RomOperator : public TimeDependentOperator const CAROM::Matrix *V_x_, const CAROM::Matrix *U_H_, const CAROM::Matrix *Hsinv_, const int myid, const bool oversampling_, const bool hyperreduce_, const bool x_base_only_, const bool use_eqp, CAROM::Vector *eqpSol, - const IntegrationRule *ir_eqp_); + const IntegrationRule *ir_eqp_, NeoHookeanModel *model_); virtual void Mult(const Vector &y, Vector &dy_dt) const; void Mult_Hyperreduced(const Vector &y, Vector &dy_dt) const; @@ -1102,7 +1104,7 @@ int main(int argc, char *argv[]) { romop = new RomOperator(&oper, soper, rvdim, rxdim, hdim, smm, w_v0, w_x0, vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, myid, - num_samples_req != -1, hyperreduce, x_base_only, use_eqp, eqpSol, ir0); + num_samples_req != -1, hyperreduce, x_base_only, use_eqp, eqpSol, ir0, model); } else { @@ -1110,7 +1112,7 @@ int main(int argc, char *argv[]) &(vx0.GetBlock(0)), &(vx0.GetBlock(1)), vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, myid, - num_samples_req != -1, hyperreduce, x_base_only, use_eqp, eqpSol, ir0); + num_samples_req != -1, hyperreduce, x_base_only, use_eqp, eqpSol, ir0, model); } // Print lifted initial energies @@ -1577,13 +1579,13 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, const CAROM::Matrix *V_x_, const CAROM::Matrix *U_H_, const CAROM::Matrix *Hsinv_, const int myid, const bool oversampling_, const bool hyperreduce_, const bool x_base_only_, const bool use_eqp, - CAROM::Vector *eqpSol, const IntegrationRule *ir_eqp_) + CAROM::Vector *eqpSol, const IntegrationRule *ir_eqp_, NeoHookeanModel *model_) : TimeDependentOperator(rxdim_ + rvdim_, 0.0), fom(fom_), fomSp(fomSp_), rxdim(rxdim_), rvdim(rvdim_), hdim(hdim_), x0(x0_), v0(v0_), v0_fom(v0_fom_), smm(smm_), V_x(*V_x_), V_v(*V_v_), U_H(U_H_), Hsinv(Hsinv_), M_hat_solver(fom_->fespace.GetComm()), oversampling(oversampling_), z(height / 2), hyperreduce(hyperreduce_), x_base_only(x_base_only_), eqp(use_eqp), - ir_eqp(ir_eqp_) + ir_eqp(ir_eqp_), model(model_) { if (!eqp) { @@ -1704,7 +1706,7 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, << fom->fespace.GetNE() << endl; // TODO: implement the one below - GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, *U_H, eqp_coef); + //GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, *U_H, eqp_coef); } } @@ -1742,7 +1744,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, - eqp_qp, ir_eqp, + eqp_qp, ir_eqp, model, V_x, x_librom, rank, resEQP); Vector recv(resEQP); diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 26bff7337..5ec41cda3 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -59,20 +59,17 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, const IntegrationRule *ir, CAROM::Matrix const &V, Vector &res) { - MFEM_ABORT("TODO") + MFEM_ABORT("TODO"); } void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, + const IntegrationRule *ir, NeoHookeanModel *model, CAROM::Matrix const &V, CAROM::Vector const &x, const int rank, Vector &res) -{ - MFEM_ABORT("TODO"); -} -/* { const int rdim = V.numColumns(); - MFEM_VERIFY(rw.size() == qp.size(), ""); + const int fomdim = V.numRows(); + MFEM_VERIFY(rw.size() == qp.size(), ""); MFEM_VERIFY(x.dim() == rdim, ""); MFEM_VERIFY(V.numRows() == fesR->GetTrueVSize(), ""); @@ -86,301 +83,114 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, const FiniteElement *fe = NULL; Array vdofs; - DenseMatrix trial_vshape; - - Vector Vx; - Vector Vj; res.SetSize(rdim); res = 0.0; int eprev = -1; int dof = 0; - int spaceDim = 0; - - // Note that the alternative version VectorFEMassIntegrator_ComputeReducedEQP_Fast - // of this function is optimized by storing some intermediate computations. - - for (int i = 0; i < rw.size(); ++i) - { - const int e = qp[i] / nqe; // Element index - // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); - const IntegrationPoint &ip = ir->IntPoint(qpi); - - if (e != eprev) // Update element transformation - { - doftrans = fesR->GetElementVDofs(e, vdofs); - fe = fesR->GetFE(e); - eltrans = fesR->GetElementTransformation(e); - - if (doftrans) - { - MFEM_ABORT("TODO"); - } - - dof = fe->GetDof(); - spaceDim = eltrans->GetSpaceDim(); - trial_vshape.SetSize(dof, spaceDim); - Vx.SetSize(spaceDim); - Vj.SetSize(spaceDim); - - // Initialize nonlinear operator matrices (there is probably a better way) - int dim = fe->GetDim(); - DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DenseMatrix Jpt(dim); - DenseMatrix P(dim); - DenseMatrix PMatI; - PMatI.UseExternalData(elfun.GetData(), dof, dim); // Q: How to replace? - // PMatO.SetSize(dof, dim); // Q: should this be here? - - eprev = e; - } - - // Integrate at the current point - eltrans->SetIntPoint(&ip); - fe->CalcVShape(*eltrans, trial_vshape); - - double w = eltrans->Weight() * rw[i]; // using rw[i] instead of ip.weight - - // Compute action of nonlinear operator - CalcInverse(eltrans->Jacobian(), Jrt); - fe.CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); - // AddMultABt(DS, P, PMatO); //Q: Should this be here? - - // Lift Vx at ip. - Vx = 0.0; - for (int k = 0; k < dof; ++k) - { - const int dofk = (vdofs[k] >= 0) ? vdofs[k] : -1 - vdofs[k]; - - double Vx_k = 0.0; - for (int j = 0; j < rdim; ++j) - { - Vx_k += V(dofk, j) * x(j); - } - - if (vdofs[k] < 0) // Q:why this? - Vx_k = -Vx_k; + int dim = 0; - for (int j = 0; j < spaceDim; ++j) - Vx[j] += Vx_k * trial_vshape(k, j); - } - - Vj = 0.0; - for (int k = 0; k < spaceDim; ++k) - { - double Vjk = 0.0; - for (int l = 0; l < dof; ++l) - { - // Q: Should these ones be the same? - const int dofl = (vdofs[l] >= 0) ? vdofs[l] : -1 - vdofs[l]; - const double s = (vdofs[l] >= 0) ? 1.0 : -1.0; - Vjk += s * V(dofl, j) * trial_vshape(l, k); - } - - Vj[k] = Vjk; - } - - // Calculate r[i] = Vj^T * P * Vx - // Perform the vector-matrix-vector multiplication: a^T * B * c - Vector temp(dim); - P.Mult(Vx, temp); - w *= Vj * temp; - - // Perform final scaling - for (int j = 0; j < rdim; ++j) - { - res[j] += w - } - } -} -*/ -/* -void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, - std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, Coefficient *Q, - CAROM::Matrix const &V, CAROM::Vector const &x, const int rank, Vector &res) -{ - - const int rdim = V.numColumns(); - MFEM_VERIFY(V.numRows() == fesR->GetTrueVSize(), ""); - MFEM_VERIFY(rw.size() == qp.size(), ""); - - const int ne = fesR->GetNE(); - const int nqe = ir->GetWeights().Size(); - - ElementTransformation *eltrans; - DofTransformation *doftrans; - const FiniteElement *fe = NULL; - Array vdofs; - - DenseMatrix trial_vshape; + // Get prolongation matrix + const Operator *P = fesR->GetProlongationMatrix(); - Vector Vx; - res.SetSize(rdim * rdim * rw.size()); - res = 0.0; + // Vectors to be prolongated + CAROM::Vector* Vx_librom = new CAROM::Vector(fomdim, false); + Vector *Vx = new Vector(&((*Vx_librom)(0)), fomdim); + Vector vj(fomdim); - // For the parallel case, we must get all DOFs of V on sampled elements. - CAROM::Matrix Vs; + // Prolongated vectors + Vector p_Vx(P->Height()); + Vector p_vj(P->Height()); - // Since V only has rows for true DOFs, we use a ParGridFunction to get all DOFs. - int eprev = -1; - int dof = 0; - int elemCount = 0; + // Element vectors + Vector Vx_e; + Vector vj_e; - // First, find all sampled elements. - for (int i = 0; i < rw.size(); ++i) - { - const int e = qp[i] / nqe; // Element index - - if (e != eprev) // Update element transformation - { - doftrans = fesR->GetElementVDofs(e, vdofs); - if (dof > 0) - { - MFEM_VERIFY(dof == vdofs.Size(), "All elements must have same DOF size"); - } - dof = vdofs.Size(); - eprev = e; - elemCount++; - } - } - - // Now set Vs. - // Note that, ideally, these FOM data structures and operations should be - // avoided, but this only done in setup. - Vs.setSize(elemCount * dof, rdim); - ParGridFunction v_gf(fesR); - Vector vtrue(fesR->GetTrueVSize()); + // Lift x and prolongate result + V.mult(x, Vx_librom); + P->Mult(*Vx, p_Vx); + // For every basis vector for (int j = 0; j < rdim; ++j) - { - eprev = -1; - elemCount = 0; - - for (int i = 0; i < vtrue.Size(); ++i) - vtrue[i] = V(i, j); - v_gf.SetFromTrueDofs(vtrue); - - for (int i = 0; i < rw.size(); ++i) + { + // Get basis vector and prolongate + for (int k = 0; k < V.numRows(); ++k) + vj[k] = V(k, j); + P->Mult(vj, p_vj); + res[j] = 0.0; + + // For every quadrature weight + for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 { const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); if (e != eprev) // Update element transformation { doftrans = fesR->GetElementVDofs(e, vdofs); + fe = fesR->GetFE(e); + eltrans = fesR->GetElementTransformation(e); - for (int k = 0; k < dof; ++k) - { // Q: Why this code? - const int dofk = (vdofs[k] >= 0) ? vdofs[k] : -1 - vdofs[k]; - const double vk = (vdofs[k] >= 0) ? v_gf[dofk] : -v_gf[dofk]; - Vs((elemCount * dof) + k, j) = vk; - } + dof = fe->GetDof(); // Get number of dofs in element + dim = fe->GetDim(); - eprev = e; - elemCount++; - } - } - } - - eprev = -1; - elemCount = 0; - int spaceDim = 0; - - for (int i = 0; i < rw.size(); ++i) - { - const int e = qp[i] / nqe; // Element index - // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); - const IntegrationPoint &ip = ir->IntPoint(qpi); + if (doftrans) + { + MFEM_ABORT("TODO"); + } - if (e != eprev) // Update element transformation - { - doftrans = fesR->GetElementVDofs(e, vdofs); - fe = fesR->GetFE(e); - eltrans = fesR->GetElementTransformation(e); + // Get element vectors + p_Vx.GetSubVector(vdofs, Vx_e); + p_vj.GetSubVector(vdofs, vj_e); - if (doftrans) - { - MFEM_ABORT("TODO"); + eprev = e; } - dof = fe->GetDof(); - spaceDim = eltrans->GetSpaceDim(); - trial_vshape.SetSize(dof, spaceDim); - Vx.SetSize(spaceDim); + // Integration at ip // Initialize nonlinear operator matrices (there is probably a better way) - int dim = el.GetDim(); DenseMatrix DSh(dof, dim); DenseMatrix DS(dof, dim); DenseMatrix Jrt(dim); DenseMatrix Jpt(dim); DenseMatrix P(dim); - DenseMatrix PMatI; - PMatI.UseExternalData(elfun.GetData(), dof, dim); // Q: How to replace? - // PMatO.SetSize(dof, dim); // Q: should this be here? - - eprev = e; - elemCount++; - } + DenseMatrix PMatI; // Extract element dofs + PMatI.UseExternalData(Vx_e.GetData(), dof, dim); + DenseMatrix PMatO; + Vector elvect(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); - // Integrate at the current point + model->SetTransformation(*eltrans); - eltrans->SetIntPoint(&ip); - fe->CalcVShape(*eltrans, trial_vshape); - - double w = eltrans->Weight() * rw[i]; // using rw[i] instead of ip.weight - - // Compute action of nonlinear operator - CalcInverse(eltrans.Jacobian(), Jrt); - fe.CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); + elvect = 0.0; - // AddMultABt(DS, P, PMatO); //Q: Should this be here? + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); - for (int jx = 0; jx < rdim; ++jx) - { - // Lift Vx = V_{jx} at ip, where x = e_{jx}. - Vx = 0.0; - for (int k = 0; k < dof; ++k) - { - const double Vx_k = Vs(((elemCount - 1) * dof) + k, jx); + // Get the transformation weight + double t = eltrans->Weight(); - for (int j = 0; j < spaceDim; ++j) - Vx[j] += Vx_k * trial_vshape(k, j); - } + // Compute action of nonlinear operator + CalcInverse(eltrans->Jacobian(), Jrt); + fe->CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P); + P *= (t * rw[i]); // NB: Not by ip.weight + AddMultABt(DS, P, PMatO); - for (int j = 0; j < rdim; ++j) + // Calculate r[i] = ve_j^T * elvect + for (int k = 0; k < elvect.Size(); k++) { - double rj = 0.0; - for (int k = 0; k < spaceDim; ++k) - { - double Vjk = 0.0; - for (int l = 0; l < dof; ++l) - { - Vjk += Vs(((elemCount - 1) * dof) + l, j) * trial_vshape(l, k); - } - - rj += Vx[k] * Vjk; - } - - res[j + (jx * rdim) + (i * rdim * rdim)] = w * rj; + res[j] += vj_e[k] * elvect[k]; } } } } -*/ - /* TODO if time... void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, std::vector const& qp, const IntegrationRule *ir, @@ -442,7 +252,6 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, { r[i] += ve_j[k] * elvect[k]; } - } } From bc5fbe7f93cbaafb3254bb8ea4c45a3f53b32100 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 20 Jun 2023 15:47:14 +0200 Subject: [PATCH 052/120] Running but wrong --- .../prom/nonlinear_elasticity_global_rom.cpp | 60 +++++++++++-------- .../nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index a1b0825f8..8870d6de2 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -171,6 +171,7 @@ class RomOperator : public TimeDependentOperator Vector *psp_v; mutable Vector zH; mutable CAROM::Vector zX; + mutable Vector zX_MFEM; mutable CAROM::Vector zN; const CAROM::Matrix *Hsinv; mutable CAROM::Vector *z_librom; @@ -206,6 +207,8 @@ class RomOperator : public TimeDependentOperator NeoHookeanModel *model; + + protected: CAROM::Matrix *S_hat; CAROM::Vector *S_hat_v0; @@ -1091,8 +1094,8 @@ int main(int argc, char *argv[]) { if (!use_eqp) { - // Define operator in sample space - soper = new HyperelasticOperator(*sp_XV_space, ess_tdof_list_sp, visc, mu, K); + // Define operator in sample space + soper = new HyperelasticOperator(*sp_XV_space, ess_tdof_list_sp, visc, mu, K); } else { @@ -1636,33 +1639,45 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, // Invert M_hat and store M_hat->inverse(*M_hat_inv); - if (myid == 0 && !eqp) + if (myid == 0 && hyperreduce) { const int spdim = fomSp->Height(); // Reduced height - zH.SetSize(spdim / 2); // Samples of H // Allocate auxillary vectors z.SetSize(spdim / 2); z_v.SetSize(spdim / 2); z_x.SetSize(spdim / 2); - z_librom = new CAROM::Vector(z.GetData(), z.Size(), false, false); - z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), false, false); - z_x_librom = new CAROM::Vector(z_x.GetData(), z_x.Size(), false, false); + if (!eqp) + { + zH.SetSize(spdim / 2); // Samples of H + z_librom = new CAROM::Vector(z.GetData(), z.Size(), false, false); + z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), false, false); + z_x_librom = new CAROM::Vector(z_x.GetData(), z_x.Size(), false, false); - // This is for saving the recreated predictions - psp_librom = new CAROM::Vector(spdim, false); - psp = new Vector(&((*psp_librom)(0)), spdim); + // This is for saving the recreated predictions + psp_librom = new CAROM::Vector(spdim, false); + psp = new Vector(&((*psp_librom)(0)), spdim); + + // Define sub-vectors of psp. + psp_x = new Vector(psp->GetData(), spdim / 2); + psp_v = new Vector(psp->GetData() + spdim / 2, spdim / 2); - // Define sub-vectors of psp. - psp_x = new Vector(psp->GetData(), spdim / 2); - psp_v = new Vector(psp->GetData() + spdim / 2, spdim / 2); + psp_v_librom = new CAROM::Vector(psp_v->GetData(), psp_v->Size(), false, false); + } + else + { + zX = CAROM::Vector(U_H->numColumns(), false); + zX_MFEM = Vector(&((zX)(0)), U_H->numColumns()); + z_librom = new CAROM::Vector(z.GetData(), z.Size(), false, false); + z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), true, false); + z_x_librom = new CAROM::Vector(z_x.GetData(), z_x.Size(), true, false); - psp_v_librom = new CAROM::Vector(psp_v->GetData(), psp_v->Size(), false, false); + } } const int fdim = fom->Height(); // Unreduced height - if (!hyperreduce || eqp) + if (!hyperreduce) { z.SetSize(fdim / 2); z_v.SetSize(fdim / 2); @@ -1706,7 +1721,7 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, << fom->fespace.GetNE() << endl; // TODO: implement the one below - //GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, *U_H, eqp_coef); + // GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, *U_H, eqp_coef); } } @@ -1735,18 +1750,15 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const { // Lift v-vector and save V_v.mult(v_librom, *z_v_librom); V_x.transposeMult(*z_v_librom, dx_dt_librom); - Vector resEQP; if (fastIntegration) - { - MFEM_ABORT("TODO"); - } - + { + MFEM_ABORT("TODO"); + } else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, model, V_x, x_librom, rank, resEQP); - Vector recv(resEQP); MPI_Allreduce(resEQP.GetData(), recv.GetData(), resEQP.Size(), MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); @@ -1761,7 +1773,8 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const MFEM_VERIFY(resEQP.Size() == rxdim, ""); for (int i = 0; i < rxdim; ++i) - z[i] += resEQP[i]; + zX_MFEM[i] += resEQP[i]; + V_vTU_H.mult(zX, z_librom); } else { // Lift x- and v-vector @@ -1796,7 +1809,6 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const // store dx_dt V_x_sp->transposeMult(*psp_v_librom, dx_dt_librom); } - if (fomSp->viscosity != 0.0) { // Apply S^, the reduced S operator, to v diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 5ec41cda3..33a1774cc 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -94,7 +94,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, const Operator *P = fesR->GetProlongationMatrix(); // Vectors to be prolongated - CAROM::Vector* Vx_librom = new CAROM::Vector(fomdim, false); + CAROM::Vector* Vx_librom = new CAROM::Vector(fomdim, true); Vector *Vx = new Vector(&((*Vx_librom)(0)), fomdim); Vector vj(fomdim); From 32e7a51074bb8e74c3223bbbf2945742adbd5a8b Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:37:38 +0200 Subject: [PATCH 053/120] Runs! --- .../prom/nonlinear_elasticity_global_rom.cpp | 9 +++-- .../nonlinear_elasticity_global_rom_eqp.hpp | 33 +++++++++++-------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 8870d6de2..0bf876cae 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1103,7 +1103,7 @@ int main(int argc, char *argv[]) } } - if (hyperreduce) // TODO: ask about whether this is needed. + if (!use_eqp) // TODO: ask about whether this is needed. { romop = new RomOperator(&oper, soper, rvdim, rxdim, hdim, smm, w_v0, w_x0, vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, myid, @@ -1168,7 +1168,7 @@ int main(int argc, char *argv[]) if (online) { - if (myid == 0 || use_eqp) + if (myid == 0) { solveTimer.Start(); ode_solver->Step(*wMFEM, t, dt_real); @@ -1757,7 +1757,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const } else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, - eqp_qp, ir_eqp, model, + eqp_qp, ir_eqp, model, x0, V_x, x_librom, rank, resEQP); Vector recv(resEQP); MPI_Allreduce(resEQP.GetData(), recv.GetData(), resEQP.Size(), MPI_DOUBLE, @@ -1815,10 +1815,10 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const S_hat->multPlus(*z_librom, v_librom, 1.0); *z_librom += *S_hat_v0; } - z.Neg(); // z = -z M_hat_inv->mult(*z_librom, dv_dt_librom); // to invert reduced mass matrix operator. + } void RomOperator::Mult_FullOrder(const Vector &vx, Vector &dvx_dt) const @@ -1864,7 +1864,6 @@ void RomOperator::Mult_FullOrder(const Vector &vx, Vector &dvx_dt) const void RomOperator::Mult(const Vector &vx, Vector &dvx_dt) const { - if (hyperreduce) Mult_Hyperreduced(vx, dvx_dt); else diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 33a1774cc..b505ae144 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -64,12 +64,12 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V, CAROM::Vector const &x, const int rank, Vector &res) { const int rdim = V.numColumns(); const int fomdim = V.numRows(); - MFEM_VERIFY(rw.size() == qp.size(), ""); + MFEM_VERIFY(rw.size() == qp.size(), ""); MFEM_VERIFY(x.dim() == rdim, ""); MFEM_VERIFY(V.numRows() == fesR->GetTrueVSize(), ""); @@ -94,8 +94,9 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, const Operator *P = fesR->GetProlongationMatrix(); // Vectors to be prolongated - CAROM::Vector* Vx_librom = new CAROM::Vector(fomdim, true); - Vector *Vx = new Vector(&((*Vx_librom)(0)), fomdim); + CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); + Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); + Vector Vx(fomdim); Vector vj(fomdim); // Prolongated vectors @@ -106,9 +107,10 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, Vector Vx_e; Vector vj_e; - // Lift x and prolongate result - V.mult(x, Vx_librom); - P->Mult(*Vx, p_Vx); + // Lift x, add x0 and prolongate result + V.mult(x, Vx_librom_temp); + add(*Vx_temp, *x0, Vx); + P->Mult(Vx, p_Vx); // For every basis vector for (int j = 0; j < rdim; ++j) @@ -120,6 +122,8 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, P->Mult(vj, p_vj); res[j] = 0.0; + eprev = -1; + // For every quadrature weight for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 { @@ -145,8 +149,10 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, // Get element vectors p_Vx.GetSubVector(vdofs, Vx_e); p_vj.GetSubVector(vdofs, vj_e); - eprev = e; + + + } // Integration at ip @@ -156,19 +162,18 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, DenseMatrix DS(dof, dim); DenseMatrix Jrt(dim); DenseMatrix Jpt(dim); - DenseMatrix P(dim); + DenseMatrix P_f(dim); DenseMatrix PMatI; // Extract element dofs PMatI.UseExternalData(Vx_e.GetData(), dof, dim); DenseMatrix PMatO; Vector elvect(dof * dim); PMatO.UseExternalData(elvect.GetData(), dof, dim); - model->SetTransformation(*eltrans); - elvect = 0.0; // Set integration point in the element transformation eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); // Get the transformation weight double t = eltrans->Weight(); @@ -178,9 +183,9 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, fe->CalcDShape(ip, DSh); Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); - P *= (t * rw[i]); // NB: Not by ip.weight - AddMultABt(DS, P, PMatO); + model->EvalP(Jpt, P_f); + P_f *= (t * rw[i]); // NB: Not by ip.weight + AddMultABt(DS, P_f, PMatO); // Calculate r[i] = ve_j^T * elvect for (int k = 0; k < elvect.Size(); k++) From e965d7671de26656414f33082301299cffa3c00d Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 20 Jun 2023 21:54:12 +0200 Subject: [PATCH 054/120] Fixed some issues --- .../prom/nonlinear_elasticity_global_rom.cpp | 20 ++++++------------ .../nonlinear_elasticity_global_rom_eqp.hpp | 21 ++++++++++--------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 0bf876cae..ec472fc01 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -207,8 +207,6 @@ class RomOperator : public TimeDependentOperator NeoHookeanModel *model; - - protected: CAROM::Matrix *S_hat; CAROM::Vector *S_hat_v0; @@ -917,8 +915,8 @@ int main(int argc, char *argv[]) { // EQP setup eqpSol = new CAROM::Vector(ir0->GetNPoints() * fespace.GetNE(), true); - SetupEQP_snapshots(ir0, myid, &fespace, nsets, H_librom, - GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "H"), + SetupEQP_snapshots(ir0, myid, &fespace, nsets, BV_librom, + GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "X"), preconditionNNLS, tolNNLS, maxNNLSnnz, model, *eqpSol); if (writeSampleMesh) @@ -1643,14 +1641,13 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, { const int spdim = fomSp->Height(); // Reduced height - // Allocate auxillary vectors z.SetSize(spdim / 2); z_v.SetSize(spdim / 2); z_x.SetSize(spdim / 2); if (!eqp) { - zH.SetSize(spdim / 2); // Samples of H + zH.SetSize(spdim / 2); // Samples of H z_librom = new CAROM::Vector(z.GetData(), z.Size(), false, false); z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), false, false); z_x_librom = new CAROM::Vector(z_x.GetData(), z_x.Size(), false, false); @@ -1667,12 +1664,9 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, } else { - zX = CAROM::Vector(U_H->numColumns(), false); - zX_MFEM = Vector(&((zX)(0)), U_H->numColumns()); + z.SetSize(rvdim); z_librom = new CAROM::Vector(z.GetData(), z.Size(), false, false); z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), true, false); - z_x_librom = new CAROM::Vector(z_x.GetData(), z_x.Size(), true, false); - } } @@ -1758,7 +1752,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, model, x0, - V_x, x_librom, rank, resEQP); + V_x, V_v, x_librom, rank, resEQP); Vector recv(resEQP); MPI_Allreduce(resEQP.GetData(), recv.GetData(), resEQP.Size(), MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); @@ -1773,8 +1767,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const MFEM_VERIFY(resEQP.Size() == rxdim, ""); for (int i = 0; i < rxdim; ++i) - zX_MFEM[i] += resEQP[i]; - V_vTU_H.mult(zX, z_librom); + z[i] += resEQP[i]; } else { // Lift x- and v-vector @@ -1818,7 +1811,6 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const z.Neg(); // z = -z M_hat_inv->mult(*z_librom, dv_dt_librom); // to invert reduced mass matrix operator. - } void RomOperator::Mult_FullOrder(const Vector &vx, Vector &dvx_dt) const diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index b505ae144..49bdef368 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -65,13 +65,14 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, - CAROM::Matrix const &V, CAROM::Vector const &x, const int rank, Vector &res) + CAROM::Matrix const &V_x,CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res) { - const int rdim = V.numColumns(); - const int fomdim = V.numRows(); + const int rxdim = V_x.numColumns(); + const int rvdim = V_v.numColumns(); + const int fomdim = V_v.numRows(); MFEM_VERIFY(rw.size() == qp.size(), ""); - MFEM_VERIFY(x.dim() == rdim, ""); - MFEM_VERIFY(V.numRows() == fesR->GetTrueVSize(), ""); + MFEM_VERIFY(x.dim() == rxdim, ""); + MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); MFEM_VERIFY(rank == 0, "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); @@ -83,7 +84,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, const FiniteElement *fe = NULL; Array vdofs; - res.SetSize(rdim); + res.SetSize(rxdim); res = 0.0; int eprev = -1; @@ -108,17 +109,17 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, Vector vj_e; // Lift x, add x0 and prolongate result - V.mult(x, Vx_librom_temp); + V_x.mult(x, Vx_librom_temp); add(*Vx_temp, *x0, Vx); P->Mult(Vx, p_Vx); // For every basis vector - for (int j = 0; j < rdim; ++j) + for (int j = 0; j < rvdim; ++j) { // Get basis vector and prolongate - for (int k = 0; k < V.numRows(); ++k) - vj[k] = V(k, j); + for (int k = 0; k < V_v.numRows(); ++k) + vj[k] = V_v(k, j); P->Mult(vj, p_vj); res[j] = 0.0; From 0f9992dcbdc0154f5165412632ce99e487cfe8d3 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 21 Jun 2023 20:04:51 +0200 Subject: [PATCH 055/120] rank-myid fix --- examples/prom/nonlinear_elasticity_global_rom.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index ec472fc01..a4ae48c1b 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1586,7 +1586,7 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, smm(smm_), V_x(*V_x_), V_v(*V_v_), U_H(U_H_), Hsinv(Hsinv_), M_hat_solver(fom_->fespace.GetComm()), oversampling(oversampling_), z(height / 2), hyperreduce(hyperreduce_), x_base_only(x_base_only_), eqp(use_eqp), - ir_eqp(ir_eqp_), model(model_) + ir_eqp(ir_eqp_), model(model_), rank(myid) { if (!eqp) { From 9bd2226066fb86008d7e384e0375bd825729b541 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 22 Jun 2023 12:53:10 +0200 Subject: [PATCH 056/120] Syntax improvements --- .../nonlinear_elasticity_global_rom_eqp.hpp | 56 +++++++++---------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 49bdef368..34ac6d79c 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -130,7 +130,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, { const int e = qp[i] / nqe; // Element index // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); + const int qpi = qp[i] - (e * nqe); const IntegrationPoint &ip = ir->IntPoint(qpi); if (e != eprev) // Update element transformation @@ -152,8 +152,6 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, p_vj.GetSubVector(vdofs, vj_e); eprev = e; - - } // Integration at ip @@ -185,7 +183,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); model->EvalP(Jpt, P_f); - P_f *= (t * rw[i]); // NB: Not by ip.weight + P_f *= (t * rw[i]); // NB: Not by ip.weight //Or is it rw[qpi]? AddMultABt(DS, P_f, PMatO); // Calculate r[i] = ve_j^T * elvect @@ -316,26 +314,26 @@ void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, // Compute EQP solution from constraints on snapshots. void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, - ParFiniteElementSpace *fespace_H, - const int nsets, const CAROM::Matrix *BH, - const CAROM::Matrix *BH_snapshots, + ParFiniteElementSpace *fespace_X, + const int nsets, const CAROM::Matrix *BV, + const CAROM::Matrix *BX_snapshots, const bool precondition, const double nnls_tol, const int maxNNLSnnz, NeoHookeanModel *model, CAROM::Vector &sol) { const int nqe = ir0->GetNPoints(); - const int ne = fespace_H->GetNE(); - const int NB = BH->numColumns(); + const int ne = fespace_X->GetNE(); + const int NB = BV->numColumns(); const int NQ = ne * nqe; - const int nsnap = BH_snapshots->numColumns(); + const int nsnap = BX_snapshots->numColumns(); - MFEM_VERIFY(nsnap == BH_snapshots->numColumns() || - nsnap + nsets == BH_snapshots->numColumns(), // Q: nsets? + MFEM_VERIFY(nsnap == BX_snapshots->numColumns() || + nsnap + nsets == BX_snapshots->numColumns(), // Q: nsets? ""); - MFEM_VERIFY(BH->numRows() == BH_snapshots->numRows(), ""); - MFEM_VERIFY(BH->numRows() == fespace_H->GetTrueVSize(), ""); + MFEM_VERIFY(BV->numRows() == BX_snapshots->numRows(), ""); + MFEM_VERIFY(BV->numRows() == fespace_X->GetTrueVSize(), ""); - const bool skipFirstW = (nsnap + nsets == BH_snapshots->numColumns()); + const bool skipFirstW = (nsnap + nsets == BX_snapshots->numColumns()); // Compute G of size (NB * nsnap) x NQ, but only store its transpose Gt. CAROM::Matrix Gt(NQ, NB * nsnap, true); @@ -346,8 +344,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, // with respect to the integration rule weight at that point, // where the "exact" quadrature solution is ir0->GetWeights(). - Vector h_i(BH_snapshots->numRows()); - Vector v_j(BH->numRows()); + Vector x_i(BX_snapshots->numRows()); + Vector v_j(BV->numRows()); Vector r(nqe); @@ -360,7 +358,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } // Get prolongation matrix - const Operator *P = fespace_H->GetProlongationMatrix(); + const Operator *P = fespace_X->GetProlongationMatrix(); if (!P) { MFEM_ABORT("P is null, generalize to serial case") @@ -370,15 +368,15 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, for (int i = 0; i < nsnap; ++i) { // Set the sampled dofs from the snapshot matrix - for (int j = 0; j < BH_snapshots->numRows(); ++j) - h_i[j] = (*BH_snapshots)(j, i); + for (int j = 0; j < BX_snapshots->numRows(); ++j) + x_i[j] = (*BX_snapshots)(j, i); // Get prolongated dofs - Vector ph_i; + Vector px_i; Vector elfun; - ph_i.SetSize(P->Height()); - P->Mult(h_i, ph_i); + px_i.SetSize(P->Height()); + P->Mult(x_i, px_i); if (skipFirstW && i > 0 && i % nsnapPerSet == 0) skip++; @@ -388,8 +386,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, { // Get basis vector - for (int k = 0; k < BH->numRows(); ++k) - v_j[k] = (*BH)(k, j); + for (int k = 0; k < BV->numRows(); ++k) + v_j[k] = (*BV)(k, j); // Get prolongated dofs Vector pv_j; @@ -404,10 +402,10 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, { // Get element and its dofs and transformation. Array vdofs; - DofTransformation *doftrans = fespace_H->GetElementVDofs(e, vdofs); - const FiniteElement &fe = *fespace_H->GetFE(e); - ElementTransformation *eltrans = fespace_H->GetElementTransformation(e); - ph_i.GetSubVector(vdofs, elfun); + DofTransformation *doftrans = fespace_X->GetElementVDofs(e, vdofs); + const FiniteElement &fe = *fespace_X->GetFE(e); + ElementTransformation *eltrans = fespace_X->GetElementTransformation(e); + px_i.GetSubVector(vdofs, elfun); pv_j.GetSubVector(vdofs, ve_j); if (doftrans) { From 77d5b21aab1c11e6fc1fcb7596a8b652ac6fecf7 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 22 Jun 2023 16:07:30 +0200 Subject: [PATCH 057/120] fixed bug for v0 --- .../prom/nonlinear_elasticity_global_rom.cpp | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index a4ae48c1b..1b59a755a 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1614,9 +1614,9 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, } smm->GatherDistributedMatrixRows("X", V_x, rxdim, *V_x_sp); + // Create V_vTU_H, for hyperreduction + V_v.transposeMult(*U_H, V_vTU_H); } - // Create V_vTU_H, for hyperreduction - V_v.transposeMult(*U_H, V_vTU_H); S_hat = new CAROM::Matrix(rvdim, rvdim, false); S_hat_v0 = new CAROM::Vector(rvdim, false); @@ -1639,14 +1639,14 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, if (myid == 0 && hyperreduce) { - const int spdim = fomSp->Height(); // Reduced height - - // Allocate auxillary vectors - z.SetSize(spdim / 2); - z_v.SetSize(spdim / 2); - z_x.SetSize(spdim / 2); if (!eqp) { + const int spdim = fomSp->Height(); // Reduced height + + // Allocate auxillary vectors + z.SetSize(spdim / 2); + z_v.SetSize(spdim / 2); + z_x.SetSize(spdim / 2); zH.SetSize(spdim / 2); // Samples of H z_librom = new CAROM::Vector(z.GetData(), z.Size(), false, false); z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), false, false); @@ -1666,24 +1666,16 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, { z.SetSize(rvdim); z_librom = new CAROM::Vector(z.GetData(), z.Size(), false, false); - z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), true, false); + } } const int fdim = fom->Height(); // Unreduced height - if (!hyperreduce) + if (!hyperreduce || (eqp && myid == 0)) { - z.SetSize(fdim / 2); - z_v.SetSize(fdim / 2); - z_x.SetSize(fdim / 2); - z_librom = new CAROM::Vector(z.GetData(), z.Size(), false, false); - z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), true, false); - z_x_librom = new CAROM::Vector(z_x.GetData(), z_x.Size(), true, false); - // This is for saving the recreated predictions pfom_librom = new CAROM::Vector(fdim, false); pfom = new Vector(&((*pfom_librom)(0)), fdim); - // Define sub-vectors of pfom. pfom_x = new Vector(pfom->GetData(), fdim / 2); pfom_v = new Vector(pfom->GetData() + fdim / 2, fdim / 2); @@ -1693,6 +1685,22 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, pfom_v_librom = new CAROM::Vector(pfom_v->GetData(), pfom_v->Size(), true, false); + + // Auxiliary vectors + z_v.SetSize(fdim / 2); + z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), true, false); + + } + + if (!hyperreduce) + { + z.SetSize(fdim / 2); + + z_x.SetSize(fdim / 2); + z_librom = new CAROM::Vector(z.GetData(), z.Size(), false, false); + + z_x_librom = new CAROM::Vector(z_x.GetData(), z_x.Size(), true, false); + } if (eqp) @@ -1743,7 +1751,8 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const if (eqp) { // Lift v-vector and save V_v.mult(v_librom, *z_v_librom); - V_x.transposeMult(*z_v_librom, dx_dt_librom); + add(z_v, *v0, *pfom_v); + V_x.transposeMult(*pfom_v_librom, dx_dt_librom); Vector resEQP; if (fastIntegration) { @@ -1764,7 +1773,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const // term. Instead, the FOM computation of the nonlinear term is // approximated by the reduced quadrature rule in the FOM space. // Therefore, the residual here is of dimension rxdim. - + z=0.0; MFEM_VERIFY(resEQP.Size() == rxdim, ""); for (int i = 0; i < rxdim; ++i) z[i] += resEQP[i]; @@ -1802,6 +1811,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const // store dx_dt V_x_sp->transposeMult(*psp_v_librom, dx_dt_librom); } + if (fomSp->viscosity != 0.0) { // Apply S^, the reduced S operator, to v From 8dc6b6b60ee2d98769983610daebc53e66ac187c Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 26 Jun 2023 15:37:47 +0200 Subject: [PATCH 058/120] small fix --- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 34ac6d79c..ae1117a6f 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -65,7 +65,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, - CAROM::Matrix const &V_x,CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res) + CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res) { const int rxdim = V_x.numColumns(); const int rvdim = V_v.numColumns(); From 292001918095e1e7aeae708063c0798a97f34e9b Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 27 Jun 2023 18:30:44 +0200 Subject: [PATCH 059/120] Implemented time windowing --- .../prom/nonlinear_elasticity_global_rom.cpp | 65 ++++- .../nonlinear_elasticity_global_rom_eqp.hpp | 243 +++++++++++++----- 2 files changed, 238 insertions(+), 70 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 1b59a755a..ba10c6887 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -237,6 +237,7 @@ class RomOperator : public TimeDependentOperator virtual void Mult(const Vector &y, Vector &dy_dt) const; void Mult_Hyperreduced(const Vector &y, Vector &dy_dt) const; void Mult_FullOrder(const Vector &y, Vector &dy_dt) const; + void SetEQP(CAROM::Vector *eqpSol); CAROM::Matrix V_v, V_x, V_vTU_H; const Vector *x0; @@ -462,6 +463,9 @@ int main(int argc, char *argv[]) double tolNNLS = 1.0e-14; int maxNNLSnnz = 0; + // for time windows + int n_windows = 0; + OptionsParser args(argc, argv); args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use."); @@ -837,6 +841,9 @@ int main(int argc, char *argv[]) CAROM::Vector *eqpSol = nullptr; CAROM::Vector *eqpSol_S = nullptr; + CAROM::Vector *window_ids = nullptr; + CAROM::Vector *load_eqpsol = new CAROM::Vector(1, false); // Will be resized later + // 11. Initialize ROM operator // I guess most of this should be done on id =0 if (online) @@ -911,13 +918,23 @@ int main(int argc, char *argv[]) ir0 = &(IntRules.Get(fe.GetGeomType(), 2 * fe.GetOrder() + 3)); } + // Timewindowing setup for EQP + n_windows = 25; // TODO add command line option + int n_step = int(t_final / dt); + + if (n_windows > 1) + { + window_ids = new CAROM::Vector(n_windows + 1, false); + get_window_ids(n_step, n_windows, window_ids); + } + if (use_eqp) { // EQP setup eqpSol = new CAROM::Vector(ir0->GetNPoints() * fespace.GetNE(), true); SetupEQP_snapshots(ir0, myid, &fespace, nsets, BV_librom, GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "X"), - preconditionNNLS, tolNNLS, maxNNLSnnz, model, *eqpSol); + preconditionNNLS, tolNNLS, maxNNLSnnz, model, *eqpSol, window_ids); if (writeSampleMesh) WriteMeshEQP(pmesh, myid, ir0->GetNPoints(), *eqpSol); @@ -1160,6 +1177,9 @@ int main(int argc, char *argv[]) oper.SetTime(t); bool last_step = false; + + int current_window = 0; + for (int ti = 1; !last_step; ti++) { double dt_real = min(dt, t_final - t); @@ -1168,6 +1188,15 @@ int main(int argc, char *argv[]) { if (myid == 0) { + if (use_eqp && window_ids && current_window < n_windows && ti == window_ids->item(current_window)) + { + // Load eqp and reinitialize romoperator + cout << "Time window start at" << ti << endl; + get_EQPsol(current_window, load_eqpsol); + romop->SetEQP(load_eqpsol); + ode_solver->Init(*romop); + current_window += 1; + } solveTimer.Start(); ode_solver->Step(*wMFEM, t, dt_real); solveTimer.Stop(); @@ -1666,7 +1695,6 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, { z.SetSize(rvdim); z_librom = new CAROM::Vector(z.GetData(), z.Size(), false, false); - } } @@ -1689,18 +1717,16 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, // Auxiliary vectors z_v.SetSize(fdim / 2); z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), true, false); - } if (!hyperreduce) { z.SetSize(fdim / 2); - + z_x.SetSize(fdim / 2); z_librom = new CAROM::Vector(z.GetData(), z.Size(), false, false); - - z_x_librom = new CAROM::Vector(z_x.GetData(), z_x.Size(), true, false); + z_x_librom = new CAROM::Vector(z_x.GetData(), z_x.Size(), true, false); } if (eqp) @@ -1773,7 +1799,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const // term. Instead, the FOM computation of the nonlinear term is // approximated by the reduced quadrature rule in the FOM space. // Therefore, the residual here is of dimension rxdim. - z=0.0; + z = 0.0; MFEM_VERIFY(resEQP.Size() == rxdim, ""); for (int i = 0; i < rxdim; ++i) z[i] += resEQP[i]; @@ -1871,3 +1897,28 @@ void RomOperator::Mult(const Vector &vx, Vector &dvx_dt) const else Mult_FullOrder(vx, dvx_dt); } + +void RomOperator::SetEQP(CAROM::Vector *eqpSol) +{ + std::set elements; + const int nqe = ir_eqp->GetWeights().Size(); + eqp_rw.clear(); + eqp_qp.clear(); + + for (int i = 0; i < eqpSol->dim(); ++i) + { + if ((*eqpSol)(i) > 1.0e-12) + { + const int e = i / nqe; // Element index + elements.insert(e); + eqp_rw.push_back((*eqpSol)(i)); + eqp_qp.push_back(i); + } + } + + cout << rank << ": EQP using " << elements.size() << " elements out of " + << fom->fespace.GetNE() << endl; + + // TODO: implement the one below + // GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, *U_H, eqp_coef); +} \ No newline at end of file diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index ae1117a6f..141e14a15 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -11,7 +11,7 @@ // Functions used by mixed_nonlinear_diffusion.cpp with EQP. #include "mfem/Utilities.hpp" // #include "fem/nonlininteg.hpp" - +#include using namespace mfem; using namespace std; @@ -64,7 +64,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res) { const int rxdim = V_x.numColumns(); @@ -130,7 +130,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, { const int e = qp[i] / nqe; // Element index // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); + const int qpi = qp[i] - (e * nqe); const IntegrationPoint &ip = ir->IntPoint(qpi); if (e != eprev) // Update element transformation @@ -151,7 +151,6 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, p_Vx.GetSubVector(vdofs, Vx_e); p_vj.GetSubVector(vdofs, vj_e); eprev = e; - } // Integration at ip @@ -205,6 +204,25 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes } */ +bool fileExists(const std::string& filename) +{ + std::ifstream file(filename); + return file.good(); +} + +void save_CAROM_Vector(const CAROM::Vector& vector, const std::string& filename) { + std::ofstream file(filename); + if (file.is_open()) { + for (int i = 0; i < vector.dim(); ++i) { + file << vector(i) << "\n"; + } + file.close(); + std::cout << "Vector saved as " << filename << " successfully." << std::endl; + } else { + std::cerr << "Error: Unable to open file " << filename << " for writing." << std::endl; + } +} + void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &ve_j, NeoHookeanModel *model, Vector const &elfun, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) @@ -319,7 +337,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const CAROM::Matrix *BX_snapshots, const bool precondition, const double nnls_tol, const int maxNNLSnnz, NeoHookeanModel *model, - CAROM::Vector &sol) + CAROM::Vector &sol, CAROM::Vector *window_ids) { const int nqe = ir0->GetNPoints(); const int ne = fespace_X->GetNE(); @@ -337,6 +355,18 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, // Compute G of size (NB * nsnap) x NQ, but only store its transpose Gt. CAROM::Matrix Gt(NQ, NB * nsnap, true); + int current_size = 0; + int i_start = 0; + int outer_loop_length = 0; + if (!window_ids) + { + current_size = nsnap; + outer_loop_length = 1; + } + else + { + outer_loop_length = window_ids->dim() - 1; + } // For 0 <= j < NB, 0 <= i < nsnap, 0 <= e < ne, 0 <= m < nqe, // G(j + (i*NB), (e*nqe) + m) @@ -364,81 +394,96 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, MFEM_ABORT("P is null, generalize to serial case") } - // For every snapshot - for (int i = 0; i < nsnap; ++i) + // Outer loop for time windowing + for (int oi = 0; oi < outer_loop_length; ++oi) { - // Set the sampled dofs from the snapshot matrix - for (int j = 0; j < BX_snapshots->numRows(); ++j) - x_i[j] = (*BX_snapshots)(j, i); + if (window_ids) + { + i_start = window_ids->item(oi); + current_size = window_ids->item(oi + 1) - window_ids->item(oi) + 1; + cout << "i_start = " << i_start << endl; + Gt.setSize(NQ, NB * current_size); + } - // Get prolongated dofs - Vector px_i; - Vector elfun; + // For every snapshot in batch + for (int i = i_start; i < current_size; ++i) + { + // Set the sampled dofs from the snapshot matrix + for (int j = 0; j < BX_snapshots->numRows(); ++j) + x_i[j] = (*BX_snapshots)(j, i); - px_i.SetSize(P->Height()); - P->Mult(x_i, px_i); + // Get prolongated dofs + Vector px_i; + Vector elfun; - if (skipFirstW && i > 0 && i % nsnapPerSet == 0) - skip++; + px_i.SetSize(P->Height()); + P->Mult(x_i, px_i); - // For each basis vector - for (int j = 0; j < NB; ++j) - { + if (skipFirstW && i > 0 && i % nsnapPerSet == 0) + skip++; - // Get basis vector - for (int k = 0; k < BV->numRows(); ++k) - v_j[k] = (*BV)(k, j); + // For each basis vector + for (int j = 0; j < NB; ++j) + { - // Get prolongated dofs - Vector pv_j; - Vector ve_j; + // Get basis vector + for (int k = 0; k < BV->numRows(); ++k) + v_j[k] = (*BV)(k, j); - pv_j.SetSize(P->Height()); - P->Mult(v_j, pv_j); + // Get prolongated dofs + Vector pv_j; + Vector ve_j; - // TODO: is it better to make the element loop the outer loop? - // For each element - for (int e = 0; e < ne; ++e) - { - // Get element and its dofs and transformation. - Array vdofs; - DofTransformation *doftrans = fespace_X->GetElementVDofs(e, vdofs); - const FiniteElement &fe = *fespace_X->GetFE(e); - ElementTransformation *eltrans = fespace_X->GetElementTransformation(e); - px_i.GetSubVector(vdofs, elfun); - pv_j.GetSubVector(vdofs, ve_j); - if (doftrans) + pv_j.SetSize(P->Height()); + P->Mult(v_j, pv_j); + + // TODO: is it better to make the element loop the outer loop? + // For each element + for (int e = 0; e < ne; ++e) { - MFEM_ABORT("Doftrans is true, make corresponding edits") + // Get element and its dofs and transformation. + Array vdofs; + DofTransformation *doftrans = fespace_X->GetElementVDofs(e, vdofs); + const FiniteElement &fe = *fespace_X->GetFE(e); + ElementTransformation *eltrans = fespace_X->GetElementTransformation(e); + px_i.GetSubVector(vdofs, elfun); + pv_j.GetSubVector(vdofs, ve_j); + if (doftrans) + { + MFEM_ABORT("Doftrans is true, make corresponding edits") + } + + // Compute the row of G corresponding to element e, store in r + ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r); + + for (int m = 0; m < nqe; ++m) + Gt((e * nqe) + m, j + (i * NB)) = r[m]; } - - // Compute the row of G corresponding to element e, store in r - ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r); - - for (int m = 0; m < nqe; ++m) - Gt((e * nqe) + m, j + (i * NB)) = r[m]; } - } - if (precondition) - { - MFEM_ABORT("TODO"); - } - } // Loop (i) over snapshots + if (precondition) + { + MFEM_ABORT("TODO"); + } + } // Loop (i) over snapshots - Array const &w_el = ir0->GetWeights(); - MFEM_VERIFY(w_el.Size() == nqe, ""); + Array const &w_el = ir0->GetWeights(); + MFEM_VERIFY(w_el.Size() == nqe, ""); - CAROM::Vector w(ne * nqe, true); + CAROM::Vector w(ne * nqe, true); - for (int i = 0; i < ne; ++i) - { - for (int j = 0; j < nqe; ++j) - w((i * nqe) + j) = w_el[j]; + for (int i = 0; i < ne; ++i) + { + for (int j = 0; j < nqe; ++j) + w((i * nqe) + j) = w_el[j]; + } + + SolveNNLS(rank, nnls_tol, maxNNLSnnz, w, Gt, sol); + if (window_ids) + { + save_CAROM_Vector(sol, "sol_" + std::to_string(oi) + ".csv"); + } } - bool stop = true; - - SolveNNLS(rank, nnls_tol, maxNNLSnnz, w, Gt, sol); } void WriteMeshEQP(ParMesh *pmesh, const int myid, const int nqe, @@ -464,3 +509,75 @@ void WriteMeshEQP(ParMesh *pmesh, const int myid, const int nqe, CAROM::SampleVisualization(pmesh, elements, elements, faces, edges, vertices, "EQPvis", &elemCount); } + +// Function to compute the indices at which to switch time windows +void get_window_ids(int n_step, int n_window, CAROM::Vector* ids) +{ + int window_size = std::round(n_step / n_window); + + int res = n_step - (n_window * (window_size - 1) + 1); + int res_up = res - n_window; + int ctr = -1; + + for (int i = 1; i <= n_window + 1; ++i) + { + ids->item(i - 1) = (i - 1) * window_size + 1 - (i - 1); + if (res_up > 0) + { + if (i <= res - res_up) + { + ctr += 1; + } + if (i > 2 && i <= res_up + 1) + { + ctr += 1; + } + } + else + { + if (i <= res + 1) + { + ctr += 1; + } + } + ids->item(i - 1) += ctr; + } +} + +void load_CAROM_vector(const std::string& filename, CAROM::Vector& vector) +{ + std::ifstream file(filename); + std::string line; + std::vector data; + + while (std::getline(file, line)) + { + std::stringstream ss(line); + std::string value; + + while (std::getline(ss, value, ',')) + { + data.push_back(std::stod(value)); + } + } + + // Set the size of the vector + int size = data.size(); + vector.setSize(size); + + // Copy the data into the vector + for (int i = 0; i < size; ++i) + { + vector(i) = data[i]; + } +} + +void get_EQPsol(const int current_window, CAROM::Vector* load_eqpsol) +{ + string filename = "sol_" + std::to_string(current_window) + ".csv"; + if (fileExists(filename)) + { + load_CAROM_vector(filename, *load_eqpsol); + } + +} From 4b52e5066860ba9e9fbb876392d59ca25bcb5a73 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 27 Jun 2023 18:36:17 +0200 Subject: [PATCH 060/120] Added command line option for time windowing --- examples/prom/nonlinear_elasticity_global_rom.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index ba10c6887..6f20fa761 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -538,6 +538,8 @@ int main(int argc, char *argv[]) "Tolerance for NNLS solver."); args.AddOption(&maxNNLSnnz, "-maxnnls", "--max-nnls", "Maximum nnz for NNLS"); + args.AddOption(&n_windows, "-ntw", "--n-time-windows", + "Number of time windows. Default is 0"); args.Parse(); if (!args.Good()) @@ -919,7 +921,6 @@ int main(int argc, char *argv[]) } // Timewindowing setup for EQP - n_windows = 25; // TODO add command line option int n_step = int(t_final / dt); if (n_windows > 1) From 4d286fcfbcb97531f524ab7ca8bf583f47bc906a Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 28 Jun 2023 15:59:07 +0200 Subject: [PATCH 061/120] Added EQP command line examples --- .../prom/nonlinear_elasticity_global_rom.cpp | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 6f20fa761..42fce1ca8 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -61,6 +61,13 @@ // Relative error of ROM position (x) at t_final: 5 is 0.00188458 // Relative error of ROM velocity (v) at t_final: 5 is 0.978726 // +// Online phase with EQP hyper-reduction +// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -ntw 50 -rvdim 40 -rxdim 10 -hdim 1 -sc 1.00 +// Output message: +// Elapsed time for time integration loop 82.0641 +// Relative error of ROM position (x) at t_final: 5 is 0.000893109 +// Relative error of ROM velocity (v) at t_final: 5 is 0.741266 +// // ================================================================================= // // Sample runs and results for parametric ROM using only displacement basis @@ -96,6 +103,25 @@ // Relative error of ROM position (x) at t_final: 5 is 0.0108709 // Relative error of ROM velocity (v) at t_final: 5 is 1.30704 // +// Online phase with strong hyper reduction, using QDEIM: +// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype qdeim -rxdim 2 -hdim 4 -nsr 10 -sc 1.0 -xbo -def-ic +// Output message: +// Elapsed time for time integration loop 0.10806 +// Relative error of ROM position (x) at t_final: 5 is 0.0108709 +// Relative error of ROM velocity (v) at t_final: 5 is 1.30704 +// +// Online phase with strong hyper reduction, using QDEIM: +// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype qdeim -rxdim 2 -hdim 4 -nsr 10 -sc 1.0 -xbo -def-ic +// Output message: +// Elapsed time for time integration loop 0.10806 +// Relative error of ROM position (x) at t_final: 5 is 0.0108709 +// Relative error of ROM velocity (v) at t_final: 5 is 1.30704 +// +// Online phase with EQP hyper reduction: +// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -rxdim 2 -hdim 1 -ntw 25 -sc 1.00 -xbo -def-ic +// Elapsed time for time integration loop 0.766614 +// Relative error of ROM position (x) at t_final: 5 is 0.0161132 +// Relative error of ROM velocity (v) at t_final: 5 is 0.775545 // This example runs in parallel with MPI, by using the same number of MPI ranks // in all phases (offline, merge, online). From 2f6def4b15634ae03494dd9365c08ecff172c7a3 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 29 Jun 2023 15:00:54 +0200 Subject: [PATCH 062/120] Cleaned up code --- .../prom/nonlinear_elasticity_global_rom.cpp | 41 +------------------ .../nonlinear_elasticity_global_rom_eqp.hpp | 40 +++++++++--------- 2 files changed, 23 insertions(+), 58 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 42fce1ca8..ca67a456f 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -145,43 +145,6 @@ using namespace mfem; class ReducedSystemOperator; -/* class HyperelasticOperator : public TimeDependentOperator -{ - -protected: - ParBilinearForm *M, *S; - - CGSolver M_solver; // Krylov solver for inverting the mass matrix M - HypreSmoother M_prec; // Preconditioner for the mass matrix M - -public: - HyperelasticOperator(ParFiniteElementSpace &f, Array &ess_tdof_list_, - double visc, double mu, double K); - - /// Compute the right-hand side of the ODE system. - virtual void Mult(const Vector &vx, Vector &dvx_dt) const; - - double ElasticEnergy(const ParGridFunction &x) const; - double KineticEnergy(const ParGridFunction &v) const; - void GetElasticEnergyDensity(const ParGridFunction &x, - ParGridFunction &w) const; - - mutable Vector H_sp; - mutable Vector dvxdt_sp; - - ParFiniteElementSpace &fespace; - double viscosity; - Array ess_tdof_list; - ParNonlinearForm *H; - HyperelasticModel *model; - mutable Vector z; // auxiliary vector - mutable Vector z2; // auxiliary vector - HypreParMatrix *Mmat; // Mass matrix from ParallelAssemble() - HypreParMatrix Smat; - - virtual ~HyperelasticOperator(); -}; */ - class RomOperator : public TimeDependentOperator { private: @@ -489,7 +452,7 @@ int main(int argc, char *argv[]) double tolNNLS = 1.0e-14; int maxNNLSnnz = 0; - // for time windows + // Number of time windows int n_windows = 0; OptionsParser args(argc, argv); @@ -1217,7 +1180,7 @@ int main(int argc, char *argv[]) { if (use_eqp && window_ids && current_window < n_windows && ti == window_ids->item(current_window)) { - // Load eqp and reinitialize romoperator + // Load eqp and reinitialize ROM operator cout << "Time window start at" << ti << endl; get_EQPsol(current_window, load_eqpsol); romop->SetEQP(load_eqpsol); diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 141e14a15..fe6efe6be 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -155,7 +155,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, // Integration at ip - // Initialize nonlinear operator matrices (there is probably a better way) + // Initialize nonlinear operator matrices (this could be optimized) DenseMatrix DSh(dof, dim); DenseMatrix DS(dof, dim); DenseMatrix Jrt(dim); @@ -182,7 +182,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, Mult(DSh, Jrt, DS); MultAtB(PMatI, DS, Jpt); model->EvalP(Jpt, P_f); - P_f *= (t * rw[i]); // NB: Not by ip.weight //Or is it rw[qpi]? + P_f *= (t * rw[i]); // NB: Not by ip.weight AddMultABt(DS, P_f, PMatO); // Calculate r[i] = ve_j^T * elvect @@ -194,31 +194,34 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } } -/* TODO if time... void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, - std::vector const& qp, const IntegrationRule *ir, - Coefficient *Q, CAROM::Vector const& x, - Vector const& coef, Vector & res) + std::vector const &qp, const IntegrationRule *ir, + Coefficient *Q, CAROM::Vector const &x, + Vector const &coef, Vector &res) { - + MFEM_ABORT("TODO") } -*/ -bool fileExists(const std::string& filename) +bool fileExists(const std::string &filename) { std::ifstream file(filename); return file.good(); } -void save_CAROM_Vector(const CAROM::Vector& vector, const std::string& filename) { +void save_CAROM_Vector(const CAROM::Vector &vector, const std::string &filename) +{ std::ofstream file(filename); - if (file.is_open()) { - for (int i = 0; i < vector.dim(); ++i) { + if (file.is_open()) + { + for (int i = 0; i < vector.dim(); ++i) + { file << vector(i) << "\n"; } file.close(); std::cout << "Vector saved as " << filename << " successfully." << std::endl; - } else { + } + else + { std::cerr << "Error: Unable to open file " << filename << " for writing." << std::endl; } } @@ -477,11 +480,11 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, for (int j = 0; j < nqe; ++j) w((i * nqe) + j) = w_el[j]; } - + SolveNNLS(rank, nnls_tol, maxNNLSnnz, w, Gt, sol); if (window_ids) { - save_CAROM_Vector(sol, "sol_" + std::to_string(oi) + ".csv"); + save_CAROM_Vector(sol, "sol_" + std::to_string(oi) + ".csv"); } } } @@ -511,7 +514,7 @@ void WriteMeshEQP(ParMesh *pmesh, const int myid, const int nqe, } // Function to compute the indices at which to switch time windows -void get_window_ids(int n_step, int n_window, CAROM::Vector* ids) +void get_window_ids(int n_step, int n_window, CAROM::Vector *ids) { int window_size = std::round(n_step / n_window); @@ -544,7 +547,7 @@ void get_window_ids(int n_step, int n_window, CAROM::Vector* ids) } } -void load_CAROM_vector(const std::string& filename, CAROM::Vector& vector) +void load_CAROM_vector(const std::string &filename, CAROM::Vector &vector) { std::ifstream file(filename); std::string line; @@ -572,12 +575,11 @@ void load_CAROM_vector(const std::string& filename, CAROM::Vector& vector) } } -void get_EQPsol(const int current_window, CAROM::Vector* load_eqpsol) +void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol) { string filename = "sol_" + std::to_string(current_window) + ".csv"; if (fileExists(filename)) { load_CAROM_vector(filename, *load_eqpsol); } - } From 890d17834e74f48c1e2f7e5724a6a9d8fc05a51b Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 30 Jun 2023 13:14:57 +0200 Subject: [PATCH 063/120] Sketch for optimized EQP --- .../nonlinear_elasticity_global_rom_eqp.hpp | 197 +++++++++++++++++- 1 file changed, 193 insertions(+), 4 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index fe6efe6be..8979f65d3 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -57,9 +57,81 @@ class HyperelasticOperator : public TimeDependentOperator void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, - CAROM::Matrix const &V, Vector &res) + CAROM::Matrix const &V, Vector &coef) { - MFEM_ABORT("TODO"); + const int rvdim = V_v.numColumns(); + const int fomdim = V_v.numRows(); + MFEM_VERIFY(rw.size() == qp.size(), ""); + MFEM_VERIFY(fomdim == fesR->GetTrueVSize(), ""); + + MFEM_VERIFY(rank == 0, + "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); + + const int nqe = ir->GetWeights().Size(); + + DofTransformation *doftrans; + Array vdofs; + + int eprev = -1; + int dof = 0; + int dim = 0; + + // Get prolongation matrix + const Operator *P = fesR->GetProlongationMatrix(); + + // Vector to be prolongated + Vector vj(fomdim); + + // Prolongated vector + Vector p_vj(P->Height()); + + // Element vector + Vector vj_e; + // Get the element vector size + doftrans = fesR->GetElementVDofs(0, vdofs); + elvect_size = vdofs.size(); + + // Coefficient vector + coef.SetSize(elvect_size * rw.size() * rvdim); + coef = 0.0; + + // For every basis vector + for (int j = 0; j < rvdim; ++j) + + { + // Get basis vector and prolongate + for (int k = 0; k < V_v.numRows(); ++k) + vj[k] = V_v(k, j); + P->Mult(vj, p_vj); + + eprev = -1; + + // For every quadrature weight + for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 + { + const int e = qp[i] / nqe; // Element index + + if (e != eprev) // Update element transformation + { + doftrans = fesR->GetElementVDofs(e, vdofs); + + if (doftrans) + { + MFEM_ABORT("TODO"); + } + + // Get element vectors + p_vj.GetSubVector(vdofs, vj_e); + eprev = e; + } + + // Calculate r[i] = ve_j^T * elvect + for (int k = 0; k < elvect_size; k++) + { + coef[k + (i * elvect_size) + (j * elvect_size * rw.size())] = vj_e[k] + } + } + } } void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, @@ -196,10 +268,127 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, std::vector const &qp, const IntegrationRule *ir, - Coefficient *Q, CAROM::Vector const &x, + CAROM::Vector const &x, CAROM::Matrix const &V_x, Vector const &coef, Vector &res) { - MFEM_ABORT("TODO") + + const int rxdim = V_x.numColumns(); + const int fomdim = V_x.numRows(); + MFEM_VERIFY(rw.size() == qp.size(), ""); + MFEM_VERIFY(x.dim() == rxdim, ""); + MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); + + MFEM_VERIFY(rank == 0, + "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); + + const int nqe = ir->GetWeights().Size(); + + ElementTransformation *eltrans; + DofTransformation *doftrans; + const FiniteElement *fe = NULL; + Array vdofs; + + res.SetSize(rxdim); + res = 0.0; + + int eprev = -1; + int dof = 0; + int dim = 0; + + // Get prolongation matrix + const Operator *P = fesR->GetProlongationMatrix(); + + // Vectors to be prolongated + CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); + Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); + Vector Vx(fomdim); + + // Prolongated vectors + Vector p_Vx(P->Height()); + + // Element vectors + Vector Vx_e; + + // Lift x, add x0 and prolongate result + V_x.mult(x, Vx_librom_temp); + add(*Vx_temp, *x0, Vx); + P->Mult(Vx, p_Vx); + + // For every basis vector + for (int j = 0; j < rvdim; ++j) + + { + res[j] = 0.0; + + eprev = -1; + + // For every quadrature weight + for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 + { + const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); + + if (e != eprev) // Update element transformation + { + doftrans = fesR->GetElementVDofs(e, vdofs); + fe = fesR->GetFE(e); + eltrans = fesR->GetElementTransformation(e); + + dof = fe->GetDof(); // Get number of dofs in element + dim = fe->GetDim(); + + if (doftrans) + { + MFEM_ABORT("TODO"); + } + + // Get element vectors + p_Vx.GetSubVector(vdofs, Vx_e); + eprev = e; + } + + // Integration at ip + + // Initialize nonlinear operator matrices (this could be optimized) + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DenseMatrix Jpt(dim); + DenseMatrix P_f(dim); + DenseMatrix PMatI; // Extract element dofs + PMatI.UseExternalData(Vx_e.GetData(), dof, dim); + DenseMatrix PMatO; + Vector elvect(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); + + elvect = 0.0; + + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); + + // Get the transformation weight + double t = eltrans->Weight(); + + // Compute action of nonlinear operator + CalcInverse(eltrans->Jacobian(), Jrt); // TODO: incorporate in coefficient calculation + fe->CalcDShape(ip, DSh); // TODO: incorporate in coefficient calculation + Mult(DSh, Jrt, DS); // TODO: incorporate in coefficient calculation + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P_f); + P_f *= (t * rw[i]); // NB: Not by ip.weight + AddMultABt(DS, P_f, PMatO); + + // Calculate r[i] = ve_j^T * elvect + // coef is size len(vdofs) * rvdim * rw.size + for (int k = 0; k < elvect.Size(); k++) + { + res[j] += coef[k + (i * rw.size()) + (j * rw.size() * rvdim)] * elvect[k]; + } + } + } } bool fileExists(const std::string &filename) From 3b44aabdc4b3226e721f64d0c4c67f5d98747880 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Sun, 2 Jul 2023 14:44:53 +0200 Subject: [PATCH 064/120] Continued fast eqp implementation --- examples/prom/nonlinear_elasticity_global_rom.cpp | 15 +++++++++------ .../prom/nonlinear_elasticity_global_rom_eqp.hpp | 8 ++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index ca67a456f..81eb14de7 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -190,7 +190,7 @@ class RomOperator : public TimeDependentOperator std::vector eqp_rw; std::vector eqp_qp; Vector eqp_coef; - const bool fastIntegration = false; // TODO: implement fast integration + const bool fastIntegration = true; // TODO: implement fast integration int rank; @@ -1738,8 +1738,8 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, cout << myid << ": EQP using " << elements.size() << " elements out of " << fom->fespace.GetNE() << endl; - // TODO: implement the one below - // GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, *U_H, eqp_coef); + GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, + ir_eqp, V_v, eqp_coef); } } @@ -1772,7 +1772,10 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const Vector resEQP; if (fastIntegration) { - MFEM_ABORT("TODO"); + HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), + eqp_qp, ir_eqp, model, + x0, V_x, x_librom, + eqp_coef, rank, resEQP) } else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, @@ -1909,6 +1912,6 @@ void RomOperator::SetEQP(CAROM::Vector *eqpSol) cout << rank << ": EQP using " << elements.size() << " elements out of " << fom->fespace.GetNE() << endl; - // TODO: implement the one below - // GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, *U_H, eqp_coef); + GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, + ir_eqp, V_v, eqp_coef); } \ No newline at end of file diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 8979f65d3..7b3528b77 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -57,7 +57,7 @@ class HyperelasticOperator : public TimeDependentOperator void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, - CAROM::Matrix const &V, Vector &coef) + CAROM::Matrix const &V_v, Vector &coef) { const int rvdim = V_v.numColumns(); const int fomdim = V_v.numRows(); @@ -267,9 +267,9 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, - std::vector const &qp, const IntegrationRule *ir, - CAROM::Vector const &x, CAROM::Matrix const &V_x, - Vector const &coef, Vector &res) + std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, + const Vector *x0, CAROM::Matrix const &V_x, CAROM::Vector const &x, + Vector const &coef, const int rank, Vector &res) { const int rxdim = V_x.numColumns(); From dddaddf1f186e3e8c5816dfdaa0ade16ef7e24c8 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 3 Jul 2023 09:29:34 +0200 Subject: [PATCH 065/120] Improved regular EQP --- .../prom/nonlinear_elasticity_global_rom.cpp | 12 +-- .../nonlinear_elasticity_global_rom_eqp.hpp | 99 +++++++++++-------- 2 files changed, 64 insertions(+), 47 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 81eb14de7..5614197c4 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -190,7 +190,7 @@ class RomOperator : public TimeDependentOperator std::vector eqp_rw; std::vector eqp_qp; Vector eqp_coef; - const bool fastIntegration = true; // TODO: implement fast integration + const bool fastIntegration = false; int rank; @@ -1739,7 +1739,7 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, << fom->fespace.GetNE() << endl; GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, - ir_eqp, V_v, eqp_coef); + ir_eqp, model, V_v, rank, eqp_coef); } } @@ -1772,10 +1772,10 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const Vector resEQP; if (fastIntegration) { - HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), + HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace),eqp_rw, eqp_qp, ir_eqp, model, - x0, V_x, x_librom, - eqp_coef, rank, resEQP) + x0, V_x, V_v, x_librom, + eqp_coef, rank, resEQP); } else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, @@ -1913,5 +1913,5 @@ void RomOperator::SetEQP(CAROM::Vector *eqpSol) << fom->fespace.GetNE() << endl; GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, - ir_eqp, V_v, eqp_coef); + ir_eqp, model, V_v, rank, eqp_coef); } \ No newline at end of file diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 7b3528b77..72bb90bf5 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -56,8 +56,8 @@ class HyperelasticOperator : public TimeDependentOperator // in HyperelasticNLFIntegrator_ComputeReducedEQP. void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, - CAROM::Matrix const &V_v, Vector &coef) + const IntegrationRule *ir, NeoHookeanModel *model, + CAROM::Matrix const &V_v, const int rank, Vector &coef) { const int rvdim = V_v.numColumns(); const int fomdim = V_v.numRows(); @@ -70,6 +70,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, const int nqe = ir->GetWeights().Size(); DofTransformation *doftrans; + ElementTransformation *eltrans; Array vdofs; int eprev = -1; @@ -89,7 +90,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, Vector vj_e; // Get the element vector size doftrans = fesR->GetElementVDofs(0, vdofs); - elvect_size = vdofs.size(); + int elvect_size = vdofs.Size(); // Coefficient vector coef.SetSize(elvect_size * rw.size() * rvdim); @@ -97,7 +98,6 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, // For every basis vector for (int j = 0; j < rvdim; ++j) - { // Get basis vector and prolongate for (int k = 0; k < V_v.numRows(); ++k) @@ -110,10 +110,14 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 { const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); if (e != eprev) // Update element transformation { doftrans = fesR->GetElementVDofs(e, vdofs); + eltrans = fesR->GetElementTransformation(e); if (doftrans) { @@ -125,10 +129,15 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, eprev = e; } + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); + // Get the transformation weight + double t = eltrans->Weight(); // Calculate r[i] = ve_j^T * elvect for (int k = 0; k < elvect_size; k++) { - coef[k + (i * elvect_size) + (j * elvect_size * rw.size())] = vj_e[k] + coef[k + (i * elvect_size) + (j * elvect_size * rw.size())] = vj_e[k] * rw[i] * t; } } } @@ -185,6 +194,21 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, add(*Vx_temp, *x0, Vx); P->Mult(Vx, p_Vx); + // Initialize nonlinear operator storage + // Assuming all elements have the same dim and n_dof + fe = fesR->GetFE(0); + dof = fe->GetDof(); + dim = fe->GetDim(); + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DenseMatrix Jpt(dim); + DenseMatrix P_f(dim); + DenseMatrix PMatI; // Extract element dofs + DenseMatrix PMatO; + Vector elvect(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); + // For every basis vector for (int j = 0; j < rvdim; ++j) @@ -226,18 +250,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } // Integration at ip - - // Initialize nonlinear operator matrices (this could be optimized) - DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DenseMatrix Jpt(dim); - DenseMatrix P_f(dim); - DenseMatrix PMatI; // Extract element dofs PMatI.UseExternalData(Vx_e.GetData(), dof, dim); - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); elvect = 0.0; @@ -267,14 +280,14 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, - std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, - const Vector *x0, CAROM::Matrix const &V_x, CAROM::Vector const &x, + std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, + const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, Vector const &coef, const int rank, Vector &res) { - + const int rxdim = V_x.numColumns(); + const int rvdim = V_v.numColumns(); const int fomdim = V_x.numRows(); - MFEM_VERIFY(rw.size() == qp.size(), ""); MFEM_VERIFY(x.dim() == rxdim, ""); MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); @@ -314,16 +327,30 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes add(*Vx_temp, *x0, Vx); P->Mult(Vx, p_Vx); + // Initialize nonlinear operator storage + // Assuming all elements have the same dim and n_dof + fe = fesR->GetFE(0); + dof = fe->GetDof(); + dim = fe->GetDim(); + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DenseMatrix Jpt(dim); + DenseMatrix P_f(dim); + DenseMatrix PMatI; // Extract element dofs + DenseMatrix PMatO; + Vector elvect(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); + // For every basis vector for (int j = 0; j < rvdim; ++j) { - res[j] = 0.0; eprev = -1; // For every quadrature weight - for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 + for (int i = 0; i < qp.size(); ++i) // NOTE: i < 9 { const int e = qp[i] / nqe; // Element index // Local (element) index of the quadrature point @@ -350,42 +377,32 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes } // Integration at ip - - // Initialize nonlinear operator matrices (this could be optimized) - DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DenseMatrix Jpt(dim); - DenseMatrix P_f(dim); - DenseMatrix PMatI; // Extract element dofs - PMatI.UseExternalData(Vx_e.GetData(), dof, dim); - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); - elvect = 0.0; + PMatI.UseExternalData(Vx_e.GetData(), dof, dim); // Set integration point in the element transformation eltrans->SetIntPoint(&ip); model->SetTransformation(*eltrans); // Get the transformation weight - double t = eltrans->Weight(); + // double t = eltrans->Weight(); + // t = 1.0; //Hack // Compute action of nonlinear operator - CalcInverse(eltrans->Jacobian(), Jrt); // TODO: incorporate in coefficient calculation - fe->CalcDShape(ip, DSh); // TODO: incorporate in coefficient calculation - Mult(DSh, Jrt, DS); // TODO: incorporate in coefficient calculation + CalcInverse(eltrans->Jacobian(), Jrt); // TODO: incorporate in coefficient calculation, if it makes sense + fe->CalcDShape(ip, DSh); // TODO: incorporate in coefficient calculation, if it makes sense + Mult(DSh, Jrt, DS); // TODO: incorporate in coefficient calculation, if it makes sense + MultAtB(PMatI, DS, Jpt); model->EvalP(Jpt, P_f); - P_f *= (t * rw[i]); // NB: Not by ip.weight + // P_f *= (t * rw[i]); // NB: Not by ip.weight AddMultABt(DS, P_f, PMatO); // Calculate r[i] = ve_j^T * elvect // coef is size len(vdofs) * rvdim * rw.size for (int k = 0; k < elvect.Size(); k++) { - res[j] += coef[k + (i * rw.size()) + (j * rw.size() * rvdim)] * elvect[k]; + res[j] += coef[k + (i * qp.size()) + (j * qp.size() * rvdim)] * elvect[k]; } } } From fd508b6ea88a73924624af1986bd42b68f8f37a1 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 3 Jul 2023 14:58:23 +0200 Subject: [PATCH 066/120] V-basis speedup --- examples/prom/nonlinear_elasticity_global_rom.cpp | 2 +- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 5614197c4..02198fe27 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -190,7 +190,7 @@ class RomOperator : public TimeDependentOperator std::vector eqp_rw; std::vector eqp_qp; Vector eqp_coef; - const bool fastIntegration = false; + const bool fastIntegration = true; int rank; diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 72bb90bf5..95386b6ab 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -137,7 +137,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, // Calculate r[i] = ve_j^T * elvect for (int k = 0; k < elvect_size; k++) { - coef[k + (i * elvect_size) + (j * elvect_size * rw.size())] = vj_e[k] * rw[i] * t; + coef[k + (i * elvect_size) + (j * rw.size() * elvect_size)] = vj_e[k] * rw[i] * t; } } } @@ -197,7 +197,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, // Initialize nonlinear operator storage // Assuming all elements have the same dim and n_dof fe = fesR->GetFE(0); - dof = fe->GetDof(); + dof = fe->GetDof(); dim = fe->GetDim(); DenseMatrix DSh(dof, dim); DenseMatrix DS(dof, dim); @@ -330,7 +330,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes // Initialize nonlinear operator storage // Assuming all elements have the same dim and n_dof fe = fesR->GetFE(0); - dof = fe->GetDof(); + dof = fe->GetDof(); dim = fe->GetDim(); DenseMatrix DSh(dof, dim); DenseMatrix DS(dof, dim); @@ -402,7 +402,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes // coef is size len(vdofs) * rvdim * rw.size for (int k = 0; k < elvect.Size(); k++) { - res[j] += coef[k + (i * qp.size()) + (j * qp.size() * rvdim)] * elvect[k]; + res[j] += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * elvect[k]; } } } From 4c61100253d8b1e9243587752e6e8ea7dede5591 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 3 Jul 2023 16:04:07 +0200 Subject: [PATCH 067/120] Storing DS now --- .../prom/nonlinear_elasticity_global_rom.cpp | 7 +-- .../nonlinear_elasticity_global_rom_eqp.hpp | 48 ++++++++++++++----- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 02198fe27..d04939f60 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -190,6 +190,7 @@ class RomOperator : public TimeDependentOperator std::vector eqp_rw; std::vector eqp_qp; Vector eqp_coef; + Vector eqp_DS_coef; const bool fastIntegration = true; int rank; @@ -1739,7 +1740,7 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, << fom->fespace.GetNE() << endl; GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, - ir_eqp, model, V_v, rank, eqp_coef); + ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef); } } @@ -1775,7 +1776,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace),eqp_rw, eqp_qp, ir_eqp, model, x0, V_x, V_v, x_librom, - eqp_coef, rank, resEQP); + eqp_coef, eqp_DS_coef, rank, resEQP); } else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, @@ -1913,5 +1914,5 @@ void RomOperator::SetEQP(CAROM::Vector *eqpSol) << fom->fespace.GetNE() << endl; GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, - ir_eqp, model, V_v, rank, eqp_coef); + ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef); } \ No newline at end of file diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 95386b6ab..bc6727485 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -57,7 +57,7 @@ class HyperelasticOperator : public TimeDependentOperator void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, - CAROM::Matrix const &V_v, const int rank, Vector &coef) + CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef) { const int rvdim = V_v.numColumns(); const int fomdim = V_v.numRows(); @@ -96,6 +96,17 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, coef.SetSize(elvect_size * rw.size() * rvdim); coef = 0.0; + // Vector for storing DS + const FiniteElement *fe = fesR->GetFE(0); + dof = fe->GetDof(); + dim = fe->GetDim(); + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DS_coef.SetSize(dof * dim * rw.size() * rvdim); + DS_coef = 0.0; + int index = 0; + // For every basis vector for (int j = 0; j < rvdim; ++j) { @@ -139,6 +150,19 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, { coef[k + (i * elvect_size) + (j * rw.size() * elvect_size)] = vj_e[k] * rw[i] * t; } + + // Calculate DS and store + CalcInverse(eltrans->Jacobian(), Jrt); + fe->CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + for (int ii = 0; ii < dof; ++ii) + { + for (int jj = 0; jj < dim; ++jj) + { + index = jj + ii * dim; + DS_coef[index + (i * dof*dim) + (j * rw.size() * dof*dim)] = DS.Elem(ii,jj); + } + } } } } @@ -282,7 +306,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, - Vector const &coef, const int rank, Vector &res) + Vector const &coef, Vector const &DS_coef, const int rank, Vector &res) { const int rxdim = V_x.numColumns(); @@ -332,9 +356,8 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes fe = fesR->GetFE(0); dof = fe->GetDof(); dim = fe->GetDim(); - DenseMatrix DSh(dof, dim); + int index = 0; DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); DenseMatrix Jpt(dim); DenseMatrix P_f(dim); DenseMatrix PMatI; // Extract element dofs @@ -384,18 +407,17 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes eltrans->SetIntPoint(&ip); model->SetTransformation(*eltrans); - // Get the transformation weight - // double t = eltrans->Weight(); - // t = 1.0; //Hack - - // Compute action of nonlinear operator - CalcInverse(eltrans->Jacobian(), Jrt); // TODO: incorporate in coefficient calculation, if it makes sense - fe->CalcDShape(ip, DSh); // TODO: incorporate in coefficient calculation, if it makes sense - Mult(DSh, Jrt, DS); // TODO: incorporate in coefficient calculation, if it makes sense + for (int ii = 0; ii < dof; ++ii) + { + for (int jj = 0; jj < dim; ++jj) + { + index = jj + ii * dim; + DS.Elem(ii,jj) = DS_coef[index + (i * elvect.Size()) + (j * qp.size() * elvect.Size())]; + } + } MultAtB(PMatI, DS, Jpt); model->EvalP(Jpt, P_f); - // P_f *= (t * rw[i]); // NB: Not by ip.weight AddMultABt(DS, P_f, PMatO); // Calculate r[i] = ve_j^T * elvect From 513d82e0d92f0c3a96efb39e4a8b1efee587dac2 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 18 Jul 2023 12:57:14 +0200 Subject: [PATCH 068/120] Solved segmentation fault in non eqp hyperreduction --- examples/prom/nonlinear_elasticity_global_rom.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index d04939f60..18c4637e3 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -901,7 +901,7 @@ int main(int argc, char *argv[]) // Setup hyperreduction, using either EQP or sampled DOFs and a sample mesh. ParFiniteElementSpace *sp_XV_space; - CAROM::Matrix *Hsinv = NULL; + CAROM::Matrix *Hsinv = new CAROM::Matrix(hdim, hdim, false); // Gets resized before use. const IntegrationRule *ir0 = NULL; if (ir0 == NULL) @@ -1100,6 +1100,7 @@ int main(int argc, char *argv[]) { if (!use_eqp) { + // Define operator in sample space soper = new HyperelasticOperator(*sp_XV_space, ess_tdof_list_sp, visc, mu, K); } @@ -1799,7 +1800,8 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const z[i] += resEQP[i]; } else - { // Lift x- and v-vector + { + // Lift x- and v-vector // I.e. perform v = v0 + V_v v^, where v^ is the input V_v_sp->mult(v_librom, *z_v_librom); V_x_sp->mult(x_librom, *z_x_librom); From 7f05d94526f4f46cc2803bfd6d0958c61a3d9253 Mon Sep 17 00:00:00 2001 From: Dylan Copeland Date: Fri, 14 Jul 2023 14:53:30 -0700 Subject: [PATCH 069/120] Adding QDEIM option to some examples. --- examples/prom/nonlinear_elasticity_global_rom.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 18c4637e3..a42ed5a72 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -31,14 +31,17 @@ // // Offline phase: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.9 -id 0 +// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.9 -id 0 // // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.1 -id 1 +// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.1 -id 1 // // Merge phase: // ./nonlinear_elasticity_global_rom -merge -ns 2 -dt 0.01 -tf 5.0 // // Create FOM comparison data: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -id 2 +// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -id 2 // // Online phase with full sampling: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rvdim 40 -rxdim 10 -hdim 71 -nsr 1170 -sc 1.0 @@ -75,12 +78,15 @@ // Offline phase: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.9 -xbo -def-ic -id 0 // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.1 -xbo -def-ic -id 1 +// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.9 -xbo -def-ic -id 0 +// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.1 -xbo -def-ic -id 1 // // Merge phase: // ./nonlinear_elasticity_global_rom -merge -ns 2 -dt 0.01 -tf 5.0 -xbo // // Create FOM comparison data: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -xbo -def-ic -id 2 +// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -xbo -def-ic -id 2 // // Online phase with full sampling: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rxdim 57 -hdim 183 -nsr 1170 -sc 1.0 -xbo -def-ic From b578a64c48af5dd2e17891116f7132e6a053c512 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Sat, 22 Jul 2023 01:23:22 +0200 Subject: [PATCH 070/120] updated gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 2b3fdde15..4c981ce76 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ init_bash.sh .vscode/settings.json .vscode/libROM.code-workspace init_bash_debug_arm.sh + +# Don't track .txt files +*.txt From 64bb68e6237f47f5e32119a9ff4db66b0ad9b3d9 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Sat, 22 Jul 2023 14:19:04 +0200 Subject: [PATCH 071/120] Matrix multiplication optimization --- .../prom/nonlinear_elasticity_global_rom.cpp | 60 +++++++++++++------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index a42ed5a72..c3e00eb95 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -32,8 +32,6 @@ // Offline phase: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.9 -id 0 // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.9 -id 0 -// -// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.1 -id 1 // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.1 -id 1 // // Merge phase: @@ -159,6 +157,8 @@ class RomOperator : public TimeDependentOperator double current_dt; bool oversampling; CAROM::Matrix *V_v_sp, *V_x_sp; + CAROM::Vector *V_xTv_0_sp; + CAROM::Matrix *V_xTV_v_sp; CAROM::Matrix *V_v_sp_dist; CAROM::Vector *psp_librom, *psp_v_librom; Vector *psp; @@ -174,6 +174,9 @@ class RomOperator : public TimeDependentOperator mutable Vector z_x; mutable Vector z_v; + CAROM::Vector *v0_fom_librom, *v0_librom, *V_xTv_0; + CAROM::Matrix *V_xTV_v; + bool hyperreduce; bool x_base_only; @@ -1106,7 +1109,7 @@ int main(int argc, char *argv[]) { if (!use_eqp) { - + // Define operator in sample space soper = new HyperelasticOperator(*sp_XV_space, ess_tdof_list_sp, visc, mu, K); } @@ -1679,6 +1682,12 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), false, false); z_x_librom = new CAROM::Vector(z_x.GetData(), z_x.Size(), false, false); + v0_librom = new CAROM::Vector(v0->GetData(), v0->Size(), false, false); + V_xTV_v_sp = new CAROM::Matrix(rxdim, rvdim, false); + V_xTv_0_sp = new CAROM::Vector(spdim / 2, false); + V_x_sp->transposeMult(*V_v_sp, *V_xTV_v_sp); + V_x_sp->transposeMult(*v0_librom, V_xTv_0_sp); + // This is for saving the recreated predictions psp_librom = new CAROM::Vector(spdim, false); psp = new Vector(&((*psp_librom)(0)), spdim); @@ -1709,12 +1718,17 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, zfom_x_librom = new CAROM::Vector(zfom_x->GetData(), zfom_x->Size(), true, false); - pfom_v_librom = new CAROM::Vector(pfom_v->GetData(), pfom_v->Size(), true, + pfom_v_librom = new CAROM::Vector(pfom_v->GetData(), pfom_v->Size(), false, false); // Auxiliary vectors z_v.SetSize(fdim / 2); z_v_librom = new CAROM::Vector(z_v.GetData(), z_v.Size(), true, false); + v0_fom_librom = new CAROM::Vector(v0_fom.GetData(), v0_fom.Size(), true, false); + V_xTV_v = new CAROM::Matrix(rxdim, rvdim, false); + V_xTv_0 = new CAROM::Vector(fdim / 2, false); + V_x.transposeMult(V_v, V_xTV_v); + V_x.transposeMult(*v0_fom_librom, V_xTv_0); } if (!hyperreduce) @@ -1774,13 +1788,16 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const if (eqp) { // Lift v-vector and save - V_v.mult(v_librom, *z_v_librom); - add(z_v, *v0, *pfom_v); - V_x.transposeMult(*pfom_v_librom, dx_dt_librom); + // V_v.mult(v_librom, *z_v_librom); + // add(z_v, *v0, *pfom_v); + // V_x.transposeMult(*pfom_v_librom, dx_dt_librom); + + V_xTV_v->mult(v_librom, *pfom_v_librom); + pfom_v_librom->plus(*V_xTv_0, dx_dt_librom); Vector resEQP; if (fastIntegration) { - HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace),eqp_rw, + HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, model, x0, V_x, V_v, x_librom, eqp_coef, eqp_DS_coef, rank, resEQP); @@ -1806,14 +1823,20 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const z[i] += resEQP[i]; } else - { + { // Lift x- and v-vector // I.e. perform v = v0 + V_v v^, where v^ is the input - V_v_sp->mult(v_librom, *z_v_librom); + V_xTV_v_sp->mult(v_librom, *z_v_librom); + z_v_librom->plus(*V_xTv_0_sp, dx_dt_librom); + + // store dx_dt + //V_v_sp->mult(v_librom, *z_v_librom); + //add(z_v, *v0, *psp_v); + //V_x_sp->transposeMult(*psp_v_librom, dx_dt_librom); + V_x_sp->mult(x_librom, *z_x_librom); // Store liftings - add(z_v, *v0, *psp_v); add(z_x, *x0, *psp_x); // Hyperreduce H @@ -1835,9 +1858,6 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const // Multiply by V_v^T * U_H V_vTU_H.mult(zX, z_librom); - - // store dx_dt - V_x_sp->transposeMult(*psp_v_librom, dx_dt_librom); } if (fomSp->viscosity != 0.0) @@ -1867,11 +1887,17 @@ void RomOperator::Mult_FullOrder(const Vector &vx, Vector &dvx_dt) const // Lift the x-, and v-vectors // I.e. perform v = v0 + V_v v^, where v^ is the input + + //V_v.mult(v_librom, *z_v_librom); + //add(z_v, *v0, *pfom_v); + //V_x.transposeMult(*pfom_v_librom, dx_dt_librom); + + V_xTV_v->mult(v_librom, *pfom_v_librom); + pfom_v_librom->plus(*V_xTv_0, dx_dt_librom); + V_x.mult(x_librom, *z_x_librom); - V_v.mult(v_librom, *z_v_librom); add(z_x, *x0, *pfom_x); // Store liftings - add(z_v, *v0, *pfom_v); // Apply H to x to get z fom->H->Mult(*pfom_x, *zfom_x); @@ -1889,7 +1915,7 @@ void RomOperator::Mult_FullOrder(const Vector &vx, Vector &dvx_dt) const M_hat_inv->mult(*z_librom, dv_dt_librom); // to invert reduced mass matrix operator. - V_x.transposeMult(*pfom_v_librom, dx_dt_librom); + } void RomOperator::Mult(const Vector &vx, Vector &dvx_dt) const From f35896db99df83e42c5b0e917cd4ff56c12d50d6 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Sat, 22 Jul 2023 14:27:40 +0200 Subject: [PATCH 072/120] Tested precompute optimization --- examples/prom/nonlinear_elasticity_global_rom.cpp | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index c3e00eb95..b12e99694 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1788,9 +1788,6 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const if (eqp) { // Lift v-vector and save - // V_v.mult(v_librom, *z_v_librom); - // add(z_v, *v0, *pfom_v); - // V_x.transposeMult(*pfom_v_librom, dx_dt_librom); V_xTV_v->mult(v_librom, *pfom_v_librom); pfom_v_librom->plus(*V_xTv_0, dx_dt_librom); @@ -1827,12 +1824,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const // Lift x- and v-vector // I.e. perform v = v0 + V_v v^, where v^ is the input V_xTV_v_sp->mult(v_librom, *z_v_librom); - z_v_librom->plus(*V_xTv_0_sp, dx_dt_librom); - - // store dx_dt - //V_v_sp->mult(v_librom, *z_v_librom); - //add(z_v, *v0, *psp_v); - //V_x_sp->transposeMult(*psp_v_librom, dx_dt_librom); + z_v_librom->plus(*V_xTv_0_sp, dx_dt_librom); // Store dx/dt V_x_sp->mult(x_librom, *z_x_librom); @@ -1888,10 +1880,6 @@ void RomOperator::Mult_FullOrder(const Vector &vx, Vector &dvx_dt) const // Lift the x-, and v-vectors // I.e. perform v = v0 + V_v v^, where v^ is the input - //V_v.mult(v_librom, *z_v_librom); - //add(z_v, *v0, *pfom_v); - //V_x.transposeMult(*pfom_v_librom, dx_dt_librom); - V_xTV_v->mult(v_librom, *pfom_v_librom); pfom_v_librom->plus(*V_xTv_0, dx_dt_librom); From 48dee028e71eeee6c248af6fae65dc51aba4ef61 Mon Sep 17 00:00:00 2001 From: Axel Larsson Date: Thu, 27 Jul 2023 19:32:11 +0200 Subject: [PATCH 073/120] Numbering works now. --- examples/prom/nonlinear_elasticity_global_rom.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index b12e99694..8c9296282 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -311,6 +311,9 @@ void BasisGeneratorFinalSummary(CAROM::BasisGenerator* bg, ofstream outfile(cutoffOutputPath); double partialSum = 0.0; + stringstream prec; + int ctr = 1; + char buffer[100]; for (int sv = 0; sv < sing_vals->dim(); ++sv) { partialSum += (*sing_vals)(sv); @@ -318,7 +321,14 @@ void BasisGeneratorFinalSummary(CAROM::BasisGenerator* bg, { if (partialSum / sum > energy_fractions[i]) { - outfile << "For energy fraction: " << energy_fractions[i] << ", take first " + // Format string + prec.str(std::string()); + prec << "%." << ctr << "f"; + sprintf(buffer, prec.str().c_str(), energy_fractions[i]); + ctr++; + + // Output string + outfile << "For energy fraction: " << string(buffer) << ", take first " << sv + 1 << " of " << sing_vals->dim() << " basis vectors" << endl; energy_fractions.pop_back(); } @@ -1902,8 +1912,6 @@ void RomOperator::Mult_FullOrder(const Vector &vx, Vector &dvx_dt) const z.Neg(); // z = -z M_hat_inv->mult(*z_librom, dv_dt_librom); // to invert reduced mass matrix operator. - - } void RomOperator::Mult(const Vector &vx, Vector &dvx_dt) const From 5acac5f17e92777910772251c6f9551eab06f2c8 Mon Sep 17 00:00:00 2001 From: Axel Larsson Date: Tue, 8 Aug 2023 16:29:42 +0200 Subject: [PATCH 074/120] Testing .hpp fix again --- .../nonlinear_elasticity_global_rom_eqp.cpp | 122 ++ .../nonlinear_elasticity_global_rom_eqp.hpp | 1519 ++++++++--------- 2 files changed, 828 insertions(+), 813 deletions(-) create mode 100644 examples/prom/nonlinear_elasticity_global_rom_eqp.cpp diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.cpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.cpp new file mode 100644 index 000000000..f05788cf3 --- /dev/null +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.cpp @@ -0,0 +1,122 @@ +#include "mfem/Utilities.hpp" +// #include "fem/nonlininteg.hpp" +#include +using namespace mfem; +using namespace std; + +#include "nonlinear_elasticity_global_rom_eqp.hpp" + +// Compute coefficients of the reduced integrator with respect to inputs Q and x +// in HyperelasticNLFIntegrator_ComputeReducedEQP. +void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, + CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef) +{ + const int rvdim = V_v.numColumns(); + const int fomdim = V_v.numRows(); + MFEM_VERIFY(rw.size() == qp.size(), ""); + MFEM_VERIFY(fomdim == fesR->GetTrueVSize(), ""); + + MFEM_VERIFY(rank == 0, + "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); + + const int nqe = ir->GetWeights().Size(); + + DofTransformation *doftrans; + ElementTransformation *eltrans; + Array vdofs; + + int eprev = -1; + int dof = 0; + int dim = 0; + + // Get prolongation matrix + const Operator *P = fesR->GetProlongationMatrix(); + + // Vector to be prolongated + Vector vj(fomdim); + + // Prolongated vector + Vector p_vj(P->Height()); + + // Element vector + Vector vj_e; + // Get the element vector size + doftrans = fesR->GetElementVDofs(0, vdofs); + int elvect_size = vdofs.Size(); + + // Coefficient vector + coef.SetSize(elvect_size * rw.size() * rvdim); + coef = 0.0; + + // Vector for storing DS + const FiniteElement *fe = fesR->GetFE(0); + dof = fe->GetDof(); + dim = fe->GetDim(); + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DS_coef.SetSize(dof * dim * rw.size() * rvdim); + DS_coef = 0.0; + int index = 0; + + // For every basis vector + for (int j = 0; j < rvdim; ++j) + { + // Get basis vector and prolongate + for (int k = 0; k < V_v.numRows(); ++k) + vj[k] = V_v(k, j); + P->Mult(vj, p_vj); + + eprev = -1; + + // For every quadrature weight + for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 + { + const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); + + if (e != eprev) // Update element transformation + { + doftrans = fesR->GetElementVDofs(e, vdofs); + eltrans = fesR->GetElementTransformation(e); + + if (doftrans) + { + MFEM_ABORT("TODO"); + } + + // Get element vectors + p_vj.GetSubVector(vdofs, vj_e); + eprev = e; + } + + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); + // Get the transformation weight + double t = eltrans->Weight(); + // Calculate r[i] = ve_j^T * elvect + for (int k = 0; k < elvect_size; k++) + { + coef[k + (i * elvect_size) + (j * rw.size() * elvect_size)] = vj_e[k] * rw[i] * t; + } + + // Calculate DS and store + CalcInverse(eltrans->Jacobian(), Jrt); + fe->CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + for (int ii = 0; ii < dof; ++ii) + { + for (int jj = 0; jj < dim; ++jj) + { + index = jj + ii * dim; + DS_coef[index + (i * dof*dim) + (j * rw.size() * dof*dim)] = DS.Elem(ii,jj); + } + } + } + } +} diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index bc6727485..63cf63d67 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -1,813 +1,706 @@ -/****************************************************************************** - * - * Copyright (c) 2013-2023, Lawrence Livermore National Security, LLC - * and other libROM project developers. See the top-level COPYRIGHT - * file for details. - * - * SPDX-License-Identifier: (Apache-2.0 OR MIT) - * - *****************************************************************************/ - -// Functions used by mixed_nonlinear_diffusion.cpp with EQP. -#include "mfem/Utilities.hpp" -// #include "fem/nonlininteg.hpp" -#include -using namespace mfem; -using namespace std; - -class HyperelasticOperator : public TimeDependentOperator -{ - -protected: - ParBilinearForm *M, *S; - - CGSolver M_solver; // Krylov solver for inverting the mass matrix M - HypreSmoother M_prec; // Preconditioner for the mass matrix M - -public: - HyperelasticOperator(ParFiniteElementSpace &f, Array &ess_tdof_list_, - double visc, double mu, double K); - - /// Compute the right-hand side of the ODE system. - virtual void Mult(const Vector &vx, Vector &dvx_dt) const; - - double ElasticEnergy(const ParGridFunction &x) const; - double KineticEnergy(const ParGridFunction &v) const; - void GetElasticEnergyDensity(const ParGridFunction &x, - ParGridFunction &w) const; - - mutable Vector H_sp; - mutable Vector dvxdt_sp; - - ParFiniteElementSpace &fespace; - double viscosity; - Array ess_tdof_list; - ParNonlinearForm *H; - HyperelasticModel *model; - mutable Vector z; // auxiliary vector - mutable Vector z2; // auxiliary vector - HypreParMatrix *Mmat; // Mass matrix from ParallelAssemble() - HypreParMatrix Smat; - - virtual ~HyperelasticOperator(); -}; - -// Compute coefficients of the reduced integrator with respect to inputs Q and x -// in HyperelasticNLFIntegrator_ComputeReducedEQP. -void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, - std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, - CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef) -{ - const int rvdim = V_v.numColumns(); - const int fomdim = V_v.numRows(); - MFEM_VERIFY(rw.size() == qp.size(), ""); - MFEM_VERIFY(fomdim == fesR->GetTrueVSize(), ""); - - MFEM_VERIFY(rank == 0, - "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); - - const int nqe = ir->GetWeights().Size(); - - DofTransformation *doftrans; - ElementTransformation *eltrans; - Array vdofs; - - int eprev = -1; - int dof = 0; - int dim = 0; - - // Get prolongation matrix - const Operator *P = fesR->GetProlongationMatrix(); - - // Vector to be prolongated - Vector vj(fomdim); - - // Prolongated vector - Vector p_vj(P->Height()); - - // Element vector - Vector vj_e; - // Get the element vector size - doftrans = fesR->GetElementVDofs(0, vdofs); - int elvect_size = vdofs.Size(); - - // Coefficient vector - coef.SetSize(elvect_size * rw.size() * rvdim); - coef = 0.0; - - // Vector for storing DS - const FiniteElement *fe = fesR->GetFE(0); - dof = fe->GetDof(); - dim = fe->GetDim(); - DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DS_coef.SetSize(dof * dim * rw.size() * rvdim); - DS_coef = 0.0; - int index = 0; - - // For every basis vector - for (int j = 0; j < rvdim; ++j) - { - // Get basis vector and prolongate - for (int k = 0; k < V_v.numRows(); ++k) - vj[k] = V_v(k, j); - P->Mult(vj, p_vj); - - eprev = -1; - - // For every quadrature weight - for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 - { - const int e = qp[i] / nqe; // Element index - // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); - const IntegrationPoint &ip = ir->IntPoint(qpi); - - if (e != eprev) // Update element transformation - { - doftrans = fesR->GetElementVDofs(e, vdofs); - eltrans = fesR->GetElementTransformation(e); - - if (doftrans) - { - MFEM_ABORT("TODO"); - } - - // Get element vectors - p_vj.GetSubVector(vdofs, vj_e); - eprev = e; - } - - // Set integration point in the element transformation - eltrans->SetIntPoint(&ip); - model->SetTransformation(*eltrans); - // Get the transformation weight - double t = eltrans->Weight(); - // Calculate r[i] = ve_j^T * elvect - for (int k = 0; k < elvect_size; k++) - { - coef[k + (i * elvect_size) + (j * rw.size() * elvect_size)] = vj_e[k] * rw[i] * t; - } - - // Calculate DS and store - CalcInverse(eltrans->Jacobian(), Jrt); - fe->CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - for (int ii = 0; ii < dof; ++ii) - { - for (int jj = 0; jj < dim; ++jj) - { - index = jj + ii * dim; - DS_coef[index + (i * dof*dim) + (j * rw.size() * dof*dim)] = DS.Elem(ii,jj); - } - } - } - } -} - -void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, - std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, - CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res) -{ - const int rxdim = V_x.numColumns(); - const int rvdim = V_v.numColumns(); - const int fomdim = V_v.numRows(); - MFEM_VERIFY(rw.size() == qp.size(), ""); - MFEM_VERIFY(x.dim() == rxdim, ""); - MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); - - MFEM_VERIFY(rank == 0, - "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); - - const int nqe = ir->GetWeights().Size(); - - ElementTransformation *eltrans; - DofTransformation *doftrans; - const FiniteElement *fe = NULL; - Array vdofs; - - res.SetSize(rxdim); - res = 0.0; - - int eprev = -1; - int dof = 0; - int dim = 0; - - // Get prolongation matrix - const Operator *P = fesR->GetProlongationMatrix(); - - // Vectors to be prolongated - CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); - Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); - Vector Vx(fomdim); - Vector vj(fomdim); - - // Prolongated vectors - Vector p_Vx(P->Height()); - Vector p_vj(P->Height()); - - // Element vectors - Vector Vx_e; - Vector vj_e; - - // Lift x, add x0 and prolongate result - V_x.mult(x, Vx_librom_temp); - add(*Vx_temp, *x0, Vx); - P->Mult(Vx, p_Vx); - - // Initialize nonlinear operator storage - // Assuming all elements have the same dim and n_dof - fe = fesR->GetFE(0); - dof = fe->GetDof(); - dim = fe->GetDim(); - DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DenseMatrix Jpt(dim); - DenseMatrix P_f(dim); - DenseMatrix PMatI; // Extract element dofs - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); - - // For every basis vector - for (int j = 0; j < rvdim; ++j) - - { - // Get basis vector and prolongate - for (int k = 0; k < V_v.numRows(); ++k) - vj[k] = V_v(k, j); - P->Mult(vj, p_vj); - res[j] = 0.0; - - eprev = -1; - - // For every quadrature weight - for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 - { - const int e = qp[i] / nqe; // Element index - // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); - const IntegrationPoint &ip = ir->IntPoint(qpi); - - if (e != eprev) // Update element transformation - { - doftrans = fesR->GetElementVDofs(e, vdofs); - fe = fesR->GetFE(e); - eltrans = fesR->GetElementTransformation(e); - - dof = fe->GetDof(); // Get number of dofs in element - dim = fe->GetDim(); - - if (doftrans) - { - MFEM_ABORT("TODO"); - } - - // Get element vectors - p_Vx.GetSubVector(vdofs, Vx_e); - p_vj.GetSubVector(vdofs, vj_e); - eprev = e; - } - - // Integration at ip - PMatI.UseExternalData(Vx_e.GetData(), dof, dim); - - elvect = 0.0; - - // Set integration point in the element transformation - eltrans->SetIntPoint(&ip); - model->SetTransformation(*eltrans); - - // Get the transformation weight - double t = eltrans->Weight(); - - // Compute action of nonlinear operator - CalcInverse(eltrans->Jacobian(), Jrt); - fe->CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P_f); - P_f *= (t * rw[i]); // NB: Not by ip.weight - AddMultABt(DS, P_f, PMatO); - - // Calculate r[i] = ve_j^T * elvect - for (int k = 0; k < elvect.Size(); k++) - { - res[j] += vj_e[k] * elvect[k]; - } - } - } -} - -void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, - std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, - const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, - Vector const &coef, Vector const &DS_coef, const int rank, Vector &res) -{ - - const int rxdim = V_x.numColumns(); - const int rvdim = V_v.numColumns(); - const int fomdim = V_x.numRows(); - MFEM_VERIFY(x.dim() == rxdim, ""); - MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); - - MFEM_VERIFY(rank == 0, - "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); - - const int nqe = ir->GetWeights().Size(); - - ElementTransformation *eltrans; - DofTransformation *doftrans; - const FiniteElement *fe = NULL; - Array vdofs; - - res.SetSize(rxdim); - res = 0.0; - - int eprev = -1; - int dof = 0; - int dim = 0; - - // Get prolongation matrix - const Operator *P = fesR->GetProlongationMatrix(); - - // Vectors to be prolongated - CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); - Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); - Vector Vx(fomdim); - - // Prolongated vectors - Vector p_Vx(P->Height()); - - // Element vectors - Vector Vx_e; - - // Lift x, add x0 and prolongate result - V_x.mult(x, Vx_librom_temp); - add(*Vx_temp, *x0, Vx); - P->Mult(Vx, p_Vx); - - // Initialize nonlinear operator storage - // Assuming all elements have the same dim and n_dof - fe = fesR->GetFE(0); - dof = fe->GetDof(); - dim = fe->GetDim(); - int index = 0; - DenseMatrix DS(dof, dim); - DenseMatrix Jpt(dim); - DenseMatrix P_f(dim); - DenseMatrix PMatI; // Extract element dofs - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); - - // For every basis vector - for (int j = 0; j < rvdim; ++j) - - { - - eprev = -1; - - // For every quadrature weight - for (int i = 0; i < qp.size(); ++i) // NOTE: i < 9 - { - const int e = qp[i] / nqe; // Element index - // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); - const IntegrationPoint &ip = ir->IntPoint(qpi); - - if (e != eprev) // Update element transformation - { - doftrans = fesR->GetElementVDofs(e, vdofs); - fe = fesR->GetFE(e); - eltrans = fesR->GetElementTransformation(e); - - dof = fe->GetDof(); // Get number of dofs in element - dim = fe->GetDim(); - - if (doftrans) - { - MFEM_ABORT("TODO"); - } - - // Get element vectors - p_Vx.GetSubVector(vdofs, Vx_e); - eprev = e; - } - - // Integration at ip - elvect = 0.0; - PMatI.UseExternalData(Vx_e.GetData(), dof, dim); - - // Set integration point in the element transformation - eltrans->SetIntPoint(&ip); - model->SetTransformation(*eltrans); - - for (int ii = 0; ii < dof; ++ii) - { - for (int jj = 0; jj < dim; ++jj) - { - index = jj + ii * dim; - DS.Elem(ii,jj) = DS_coef[index + (i * elvect.Size()) + (j * qp.size() * elvect.Size())]; - } - } - - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P_f); - AddMultABt(DS, P_f, PMatO); - - // Calculate r[i] = ve_j^T * elvect - // coef is size len(vdofs) * rvdim * rw.size - for (int k = 0; k < elvect.Size(); k++) - { - res[j] += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * elvect[k]; - } - } - } -} - -bool fileExists(const std::string &filename) -{ - std::ifstream file(filename); - return file.good(); -} - -void save_CAROM_Vector(const CAROM::Vector &vector, const std::string &filename) -{ - std::ofstream file(filename); - if (file.is_open()) - { - for (int i = 0; i < vector.dim(); ++i) - { - file << vector(i) << "\n"; - } - file.close(); - std::cout << "Vector saved as " << filename << " successfully." << std::endl; - } - else - { - std::cerr << "Error: Unable to open file " << filename << " for writing." << std::endl; - } -} - -void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, - Vector const &ve_j, NeoHookeanModel *model, Vector const &elfun, - FiniteElement const &fe, ElementTransformation &Trans, Vector &r) -{ - MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); - int dof = fe.GetDof(); // Get number of dofs in element - int dim = fe.GetDim(); - - // Initialize nonlinear operator matrices (there is probably a better way) - DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DenseMatrix Jpt(dim); - DenseMatrix P(dim); - DenseMatrix PMatI; // Extract element dofs - PMatI.UseExternalData(elfun.GetData(), dof, dim); - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); - - model->SetTransformation(Trans); - - // For each integration point - for (int i = 0; i < ir->GetNPoints(); i++) - { - elvect = 0.0; - // Get integration point - const IntegrationPoint &ip = ir->IntPoint(i); - - // Set integration point in the element transformation - Trans.SetIntPoint(&ip); - - // Get the transformation weight - double t = Trans.Weight(); - - // Compute action of nonlinear operator - CalcInverse(Trans.Jacobian(), Jrt); - fe.CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); - P *= t; // NB: Not by ip.weight - AddMultABt(DS, P, PMatO); - - r[i] = 0.0; - - // Calculate r[i] = ve_j^T * elvect - for (int k = 0; k < elvect.Size(); k++) - { - r[i] += ve_j[k] * elvect[k]; - } - } -} - -void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, - CAROM::Vector const &w, CAROM::Matrix &Gt, - CAROM::Vector &sol) -{ - CAROM::NNLSSolver nnls(nnls_tol, 0, maxNNLSnnz, 2); - - CAROM::Vector rhs_ub(Gt.numColumns(), false); - // G.mult(w, rhs_ub); // rhs = Gw - // rhs = Gw. Note that by using Gt and multTranspose, we do parallel communication. - Gt.transposeMult(w, rhs_ub); - - CAROM::Vector rhs_lb(rhs_ub); - CAROM::Vector rhs_Gw(rhs_ub); - - const double delta = 1.0e-11; - for (int i = 0; i < rhs_ub.dim(); ++i) - { - rhs_lb(i) -= delta; - rhs_ub(i) += delta; - } - - // nnls.normalize_constraints(Gt, rhs_lb, rhs_ub); - nnls.solve_parallel_with_scalapack(Gt, rhs_lb, rhs_ub, sol); - - int nnz = 0; - for (int i = 0; i < sol.dim(); ++i) - { - if (sol(i) != 0.0) - { - nnz++; - } - } - - cout << rank << ": Number of nonzeros in NNLS solution: " << nnz - << ", out of " << sol.dim() << endl; - - MPI_Allreduce(MPI_IN_PLACE, &nnz, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - - if (rank == 0) - cout << "Global number of nonzeros in NNLS solution: " << nnz << endl; - - // Check residual of NNLS solution - CAROM::Vector res(Gt.numColumns(), false); - Gt.transposeMult(sol, res); - - const double normGsol = res.norm(); - const double normRHS = rhs_Gw.norm(); - - res -= rhs_Gw; - const double relNorm = res.norm() / std::max(normGsol, normRHS); - cout << rank << ": relative residual norm for NNLS solution of Gs = Gw: " << relNorm << endl; -} - -// Compute EQP solution from constraints on snapshots. -void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, - ParFiniteElementSpace *fespace_X, - const int nsets, const CAROM::Matrix *BV, - const CAROM::Matrix *BX_snapshots, - const bool precondition, const double nnls_tol, - const int maxNNLSnnz, NeoHookeanModel *model, - CAROM::Vector &sol, CAROM::Vector *window_ids) -{ - const int nqe = ir0->GetNPoints(); - const int ne = fespace_X->GetNE(); - const int NB = BV->numColumns(); - const int NQ = ne * nqe; - const int nsnap = BX_snapshots->numColumns(); - - MFEM_VERIFY(nsnap == BX_snapshots->numColumns() || - nsnap + nsets == BX_snapshots->numColumns(), // Q: nsets? - ""); - MFEM_VERIFY(BV->numRows() == BX_snapshots->numRows(), ""); - MFEM_VERIFY(BV->numRows() == fespace_X->GetTrueVSize(), ""); - - const bool skipFirstW = (nsnap + nsets == BX_snapshots->numColumns()); - - // Compute G of size (NB * nsnap) x NQ, but only store its transpose Gt. - CAROM::Matrix Gt(NQ, NB * nsnap, true); - int current_size = 0; - int i_start = 0; - int outer_loop_length = 0; - if (!window_ids) - { - current_size = nsnap; - outer_loop_length = 1; - } - else - { - outer_loop_length = window_ids->dim() - 1; - } - - // For 0 <= j < NB, 0 <= i < nsnap, 0 <= e < ne, 0 <= m < nqe, - // G(j + (i*NB), (e*nqe) + m) - // is the coefficient of v_j^T M(p_i) V v_i at point m of element e, - // with respect to the integration rule weight at that point, - // where the "exact" quadrature solution is ir0->GetWeights(). - - Vector x_i(BX_snapshots->numRows()); - Vector v_j(BV->numRows()); - - Vector r(nqe); - - int skip = 0; - const int nsnapPerSet = nsnap / nsets; - if (skipFirstW) - { - MFEM_VERIFY(nsets * nsnapPerSet == nsnap, ""); - skip = 1; - } - - // Get prolongation matrix - const Operator *P = fespace_X->GetProlongationMatrix(); - if (!P) - { - MFEM_ABORT("P is null, generalize to serial case") - } - - // Outer loop for time windowing - for (int oi = 0; oi < outer_loop_length; ++oi) - { - if (window_ids) - { - i_start = window_ids->item(oi); - current_size = window_ids->item(oi + 1) - window_ids->item(oi) + 1; - cout << "i_start = " << i_start << endl; - Gt.setSize(NQ, NB * current_size); - } - - // For every snapshot in batch - for (int i = i_start; i < current_size; ++i) - { - // Set the sampled dofs from the snapshot matrix - for (int j = 0; j < BX_snapshots->numRows(); ++j) - x_i[j] = (*BX_snapshots)(j, i); - - // Get prolongated dofs - Vector px_i; - Vector elfun; - - px_i.SetSize(P->Height()); - P->Mult(x_i, px_i); - - if (skipFirstW && i > 0 && i % nsnapPerSet == 0) - skip++; - - // For each basis vector - for (int j = 0; j < NB; ++j) - { - - // Get basis vector - for (int k = 0; k < BV->numRows(); ++k) - v_j[k] = (*BV)(k, j); - - // Get prolongated dofs - Vector pv_j; - Vector ve_j; - - pv_j.SetSize(P->Height()); - P->Mult(v_j, pv_j); - - // TODO: is it better to make the element loop the outer loop? - // For each element - for (int e = 0; e < ne; ++e) - { - // Get element and its dofs and transformation. - Array vdofs; - DofTransformation *doftrans = fespace_X->GetElementVDofs(e, vdofs); - const FiniteElement &fe = *fespace_X->GetFE(e); - ElementTransformation *eltrans = fespace_X->GetElementTransformation(e); - px_i.GetSubVector(vdofs, elfun); - pv_j.GetSubVector(vdofs, ve_j); - if (doftrans) - { - MFEM_ABORT("Doftrans is true, make corresponding edits") - } - - // Compute the row of G corresponding to element e, store in r - ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r); - - for (int m = 0; m < nqe; ++m) - Gt((e * nqe) + m, j + (i * NB)) = r[m]; - } - } - - if (precondition) - { - MFEM_ABORT("TODO"); - } - } // Loop (i) over snapshots - - Array const &w_el = ir0->GetWeights(); - MFEM_VERIFY(w_el.Size() == nqe, ""); - - CAROM::Vector w(ne * nqe, true); - - for (int i = 0; i < ne; ++i) - { - for (int j = 0; j < nqe; ++j) - w((i * nqe) + j) = w_el[j]; - } - - SolveNNLS(rank, nnls_tol, maxNNLSnnz, w, Gt, sol); - if (window_ids) - { - save_CAROM_Vector(sol, "sol_" + std::to_string(oi) + ".csv"); - } - } -} - -void WriteMeshEQP(ParMesh *pmesh, const int myid, const int nqe, - CAROM::Vector const &eqpSol) -{ - // Find the elements with quadrature points in eqpSol. - std::set elements; - Vector elemCount(pmesh->GetNE()); - elemCount = 0.0; - - for (int i = 0; i < eqpSol.dim(); ++i) - { - if (eqpSol(i) > 1.0e-12) - { - const int e = i / nqe; // Element index - elements.insert(e); - elemCount[e] += 1.0; - } - } - - // Empty sets, since EQP on samples inside elements. - std::set faces, edges, vertices; - CAROM::SampleVisualization(pmesh, elements, elements, faces, edges, - vertices, "EQPvis", &elemCount); -} - -// Function to compute the indices at which to switch time windows -void get_window_ids(int n_step, int n_window, CAROM::Vector *ids) -{ - int window_size = std::round(n_step / n_window); - - int res = n_step - (n_window * (window_size - 1) + 1); - int res_up = res - n_window; - int ctr = -1; - - for (int i = 1; i <= n_window + 1; ++i) - { - ids->item(i - 1) = (i - 1) * window_size + 1 - (i - 1); - if (res_up > 0) - { - if (i <= res - res_up) - { - ctr += 1; - } - if (i > 2 && i <= res_up + 1) - { - ctr += 1; - } - } - else - { - if (i <= res + 1) - { - ctr += 1; - } - } - ids->item(i - 1) += ctr; - } -} - -void load_CAROM_vector(const std::string &filename, CAROM::Vector &vector) -{ - std::ifstream file(filename); - std::string line; - std::vector data; - - while (std::getline(file, line)) - { - std::stringstream ss(line); - std::string value; - - while (std::getline(ss, value, ',')) - { - data.push_back(std::stod(value)); - } - } - - // Set the size of the vector - int size = data.size(); - vector.setSize(size); - - // Copy the data into the vector - for (int i = 0; i < size; ++i) - { - vector(i) = data[i]; - } -} - -void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol) -{ - string filename = "sol_" + std::to_string(current_window) + ".csv"; - if (fileExists(filename)) - { - load_CAROM_vector(filename, *load_eqpsol); - } -} +/****************************************************************************** + * + * Copyright (c) 2013-2023, Lawrence Livermore National Security, LLC + * and other libROM project developers. See the top-level COPYRIGHT + * file for details. + * + * SPDX-License-Identifier: (Apache-2.0 OR MIT) + * + *****************************************************************************/ + +// Functions used by nonlinear_elastic_global_rom.cpp with EQP. +#include "mfem/Utilities.hpp" +// #include "fem/nonlininteg.hpp" +#include +using namespace mfem; +using namespace std; + +class HyperelasticOperator : public TimeDependentOperator +{ + +protected: + ParBilinearForm *M, *S; + + CGSolver M_solver; // Krylov solver for inverting the mass matrix M + HypreSmoother M_prec; // Preconditioner for the mass matrix M + +public: + HyperelasticOperator(ParFiniteElementSpace &f, Array &ess_tdof_list_, + double visc, double mu, double K); + + /// Compute the right-hand side of the ODE system. + virtual void Mult(const Vector &vx, Vector &dvx_dt) const; + + double ElasticEnergy(const ParGridFunction &x) const; + double KineticEnergy(const ParGridFunction &v) const; + void GetElasticEnergyDensity(const ParGridFunction &x, + ParGridFunction &w) const; + + mutable Vector H_sp; + mutable Vector dvxdt_sp; + + ParFiniteElementSpace &fespace; + double viscosity; + Array ess_tdof_list; + ParNonlinearForm *H; + HyperelasticModel *model; + mutable Vector z; // auxiliary vector + mutable Vector z2; // auxiliary vector + HypreParMatrix *Mmat; // Mass matrix from ParallelAssemble() + HypreParMatrix Smat; + + virtual ~HyperelasticOperator(); +}; + +// Compute coefficients of the reduced integrator with respect to inputs Q and x +// in HyperelasticNLFIntegrator_ComputeReducedEQP. +void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, + CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef); + +void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, + CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res) +{ + const int rxdim = V_x.numColumns(); + const int rvdim = V_v.numColumns(); + const int fomdim = V_v.numRows(); + MFEM_VERIFY(rw.size() == qp.size(), ""); + MFEM_VERIFY(x.dim() == rxdim, ""); + MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); + + MFEM_VERIFY(rank == 0, + "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); + + const int nqe = ir->GetWeights().Size(); + + ElementTransformation *eltrans; + DofTransformation *doftrans; + const FiniteElement *fe = NULL; + Array vdofs; + + res.SetSize(rxdim); + res = 0.0; + + int eprev = -1; + int dof = 0; + int dim = 0; + + // Get prolongation matrix + const Operator *P = fesR->GetProlongationMatrix(); + + // Vectors to be prolongated + CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); + Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); + Vector Vx(fomdim); + Vector vj(fomdim); + + // Prolongated vectors + Vector p_Vx(P->Height()); + Vector p_vj(P->Height()); + + // Element vectors + Vector Vx_e; + Vector vj_e; + + // Lift x, add x0 and prolongate result + V_x.mult(x, Vx_librom_temp); + add(*Vx_temp, *x0, Vx); + P->Mult(Vx, p_Vx); + + // Initialize nonlinear operator storage + // Assuming all elements have the same dim and n_dof + fe = fesR->GetFE(0); + dof = fe->GetDof(); + dim = fe->GetDim(); + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DenseMatrix Jpt(dim); + DenseMatrix P_f(dim); + DenseMatrix PMatI; // Extract element dofs + DenseMatrix PMatO; + Vector elvect(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); + + // For every basis vector + for (int j = 0; j < rvdim; ++j) + + { + // Get basis vector and prolongate + for (int k = 0; k < V_v.numRows(); ++k) + vj[k] = V_v(k, j); + P->Mult(vj, p_vj); + res[j] = 0.0; + + eprev = -1; + + // For every quadrature weight + for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 + { + const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); + + if (e != eprev) // Update element transformation + { + doftrans = fesR->GetElementVDofs(e, vdofs); + fe = fesR->GetFE(e); + eltrans = fesR->GetElementTransformation(e); + + dof = fe->GetDof(); // Get number of dofs in element + dim = fe->GetDim(); + + if (doftrans) + { + MFEM_ABORT("TODO"); + } + + // Get element vectors + p_Vx.GetSubVector(vdofs, Vx_e); + p_vj.GetSubVector(vdofs, vj_e); + eprev = e; + } + + // Integration at ip + PMatI.UseExternalData(Vx_e.GetData(), dof, dim); + + elvect = 0.0; + + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); + + // Get the transformation weight + double t = eltrans->Weight(); + + // Compute action of nonlinear operator + CalcInverse(eltrans->Jacobian(), Jrt); + fe->CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P_f); + P_f *= (t * rw[i]); // NB: Not by ip.weight + AddMultABt(DS, P_f, PMatO); + + // Calculate r[i] = ve_j^T * elvect + for (int k = 0; k < elvect.Size(); k++) + { + res[j] += vj_e[k] * elvect[k]; + } + } + } +} + +void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, + std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, + const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, + Vector const &coef, Vector const &DS_coef, const int rank, Vector &res) +{ + + const int rxdim = V_x.numColumns(); + const int rvdim = V_v.numColumns(); + const int fomdim = V_x.numRows(); + MFEM_VERIFY(x.dim() == rxdim, ""); + MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); + + MFEM_VERIFY(rank == 0, + "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); + + const int nqe = ir->GetWeights().Size(); + + ElementTransformation *eltrans; + DofTransformation *doftrans; + const FiniteElement *fe = NULL; + Array vdofs; + + res.SetSize(rxdim); + res = 0.0; + + int eprev = -1; + int dof = 0; + int dim = 0; + + // Get prolongation matrix + const Operator *P = fesR->GetProlongationMatrix(); + + // Vectors to be prolongated + CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); + Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); + Vector Vx(fomdim); + + // Prolongated vectors + Vector p_Vx(P->Height()); + + // Element vectors + Vector Vx_e; + + // Lift x, add x0 and prolongate result + V_x.mult(x, Vx_librom_temp); + add(*Vx_temp, *x0, Vx); + P->Mult(Vx, p_Vx); + + // Initialize nonlinear operator storage + // Assuming all elements have the same dim and n_dof + fe = fesR->GetFE(0); + dof = fe->GetDof(); + dim = fe->GetDim(); + int index = 0; + DenseMatrix DS(dof, dim); + DenseMatrix Jpt(dim); + DenseMatrix P_f(dim); + DenseMatrix PMatI; // Extract element dofs + DenseMatrix PMatO; + Vector elvect(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); + + // For every basis vector + for (int j = 0; j < rvdim; ++j) + + { + + eprev = -1; + + // For every quadrature weight + for (int i = 0; i < qp.size(); ++i) // NOTE: i < 9 + { + const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); + + if (e != eprev) // Update element transformation + { + doftrans = fesR->GetElementVDofs(e, vdofs); + fe = fesR->GetFE(e); + eltrans = fesR->GetElementTransformation(e); + + dof = fe->GetDof(); // Get number of dofs in element + dim = fe->GetDim(); + + if (doftrans) + { + MFEM_ABORT("TODO"); + } + + // Get element vectors + p_Vx.GetSubVector(vdofs, Vx_e); + eprev = e; + } + + // Integration at ip + elvect = 0.0; + PMatI.UseExternalData(Vx_e.GetData(), dof, dim); + + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); + + for (int ii = 0; ii < dof; ++ii) + { + for (int jj = 0; jj < dim; ++jj) + { + index = jj + ii * dim; + DS.Elem(ii,jj) = DS_coef[index + (i * elvect.Size()) + (j * qp.size() * elvect.Size())]; + } + } + + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P_f); + AddMultABt(DS, P_f, PMatO); + + // Calculate r[i] = ve_j^T * elvect + // coef is size len(vdofs) * rvdim * rw.size + for (int k = 0; k < elvect.Size(); k++) + { + res[j] += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * elvect[k]; + } + } + } +} + +bool fileExists(const std::string &filename) +{ + std::ifstream file(filename); + return file.good(); +} + +void save_CAROM_Vector(const CAROM::Vector &vector, const std::string &filename) +{ + std::ofstream file(filename); + if (file.is_open()) + { + for (int i = 0; i < vector.dim(); ++i) + { + file << vector(i) << "\n"; + } + file.close(); + std::cout << "Vector saved as " << filename << " successfully." << std::endl; + } + else + { + std::cerr << "Error: Unable to open file " << filename << " for writing." << std::endl; + } +} + +void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, + Vector const &ve_j, NeoHookeanModel *model, Vector const &elfun, + FiniteElement const &fe, ElementTransformation &Trans, Vector &r) +{ + MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); + int dof = fe.GetDof(); // Get number of dofs in element + int dim = fe.GetDim(); + + // Initialize nonlinear operator matrices (there is probably a better way) + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DenseMatrix Jpt(dim); + DenseMatrix P(dim); + DenseMatrix PMatI; // Extract element dofs + PMatI.UseExternalData(elfun.GetData(), dof, dim); + DenseMatrix PMatO; + Vector elvect(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); + + model->SetTransformation(Trans); + + // For each integration point + for (int i = 0; i < ir->GetNPoints(); i++) + { + elvect = 0.0; + // Get integration point + const IntegrationPoint &ip = ir->IntPoint(i); + + // Set integration point in the element transformation + Trans.SetIntPoint(&ip); + + // Get the transformation weight + double t = Trans.Weight(); + + // Compute action of nonlinear operator + CalcInverse(Trans.Jacobian(), Jrt); + fe.CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P); + P *= t; // NB: Not by ip.weight + AddMultABt(DS, P, PMatO); + + r[i] = 0.0; + + // Calculate r[i] = ve_j^T * elvect + for (int k = 0; k < elvect.Size(); k++) + { + r[i] += ve_j[k] * elvect[k]; + } + } +} + +void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, + CAROM::Vector const &w, CAROM::Matrix &Gt, + CAROM::Vector &sol) +{ + CAROM::NNLSSolver nnls(nnls_tol, 0, maxNNLSnnz, 2); + + CAROM::Vector rhs_ub(Gt.numColumns(), false); + // G.mult(w, rhs_ub); // rhs = Gw + // rhs = Gw. Note that by using Gt and multTranspose, we do parallel communication. + Gt.transposeMult(w, rhs_ub); + + CAROM::Vector rhs_lb(rhs_ub); + CAROM::Vector rhs_Gw(rhs_ub); + + const double delta = 1.0e-11; + for (int i = 0; i < rhs_ub.dim(); ++i) + { + rhs_lb(i) -= delta; + rhs_ub(i) += delta; + } + + // nnls.normalize_constraints(Gt, rhs_lb, rhs_ub); + nnls.solve_parallel_with_scalapack(Gt, rhs_lb, rhs_ub, sol); + + int nnz = 0; + for (int i = 0; i < sol.dim(); ++i) + { + if (sol(i) != 0.0) + { + nnz++; + } + } + + cout << rank << ": Number of nonzeros in NNLS solution: " << nnz + << ", out of " << sol.dim() << endl; + + MPI_Allreduce(MPI_IN_PLACE, &nnz, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + + if (rank == 0) + cout << "Global number of nonzeros in NNLS solution: " << nnz << endl; + + // Check residual of NNLS solution + CAROM::Vector res(Gt.numColumns(), false); + Gt.transposeMult(sol, res); + + const double normGsol = res.norm(); + const double normRHS = rhs_Gw.norm(); + + res -= rhs_Gw; + const double relNorm = res.norm() / std::max(normGsol, normRHS); + cout << rank << ": relative residual norm for NNLS solution of Gs = Gw: " << relNorm << endl; +} + +// Compute EQP solution from constraints on snapshots. +void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, + ParFiniteElementSpace *fespace_X, + const int nsets, const CAROM::Matrix *BV, + const CAROM::Matrix *BX_snapshots, + const bool precondition, const double nnls_tol, + const int maxNNLSnnz, NeoHookeanModel *model, + CAROM::Vector &sol, CAROM::Vector *window_ids) +{ + const int nqe = ir0->GetNPoints(); + const int ne = fespace_X->GetNE(); + const int NB = BV->numColumns(); + const int NQ = ne * nqe; + const int nsnap = BX_snapshots->numColumns(); + + MFEM_VERIFY(nsnap == BX_snapshots->numColumns() || + nsnap + nsets == BX_snapshots->numColumns(), // Q: nsets? + ""); + MFEM_VERIFY(BV->numRows() == BX_snapshots->numRows(), ""); + MFEM_VERIFY(BV->numRows() == fespace_X->GetTrueVSize(), ""); + + const bool skipFirstW = (nsnap + nsets == BX_snapshots->numColumns()); + + // Compute G of size (NB * nsnap) x NQ, but only store its transpose Gt. + CAROM::Matrix Gt(NQ, NB * nsnap, true); + int current_size = 0; + int i_start = 0; + int outer_loop_length = 0; + if (!window_ids) + { + current_size = nsnap; + outer_loop_length = 1; + } + else + { + outer_loop_length = window_ids->dim() - 1; + } + + // For 0 <= j < NB, 0 <= i < nsnap, 0 <= e < ne, 0 <= m < nqe, + // G(j + (i*NB), (e*nqe) + m) + // is the coefficient of v_j^T M(p_i) V v_i at point m of element e, + // with respect to the integration rule weight at that point, + // where the "exact" quadrature solution is ir0->GetWeights(). + + Vector x_i(BX_snapshots->numRows()); + Vector v_j(BV->numRows()); + + Vector r(nqe); + + int skip = 0; + const int nsnapPerSet = nsnap / nsets; + if (skipFirstW) + { + MFEM_VERIFY(nsets * nsnapPerSet == nsnap, ""); + skip = 1; + } + + // Get prolongation matrix + const Operator *P = fespace_X->GetProlongationMatrix(); + if (!P) + { + MFEM_ABORT("P is null, generalize to serial case") + } + + // Outer loop for time windowing + for (int oi = 0; oi < outer_loop_length; ++oi) + { + if (window_ids) + { + i_start = window_ids->item(oi); + current_size = window_ids->item(oi + 1) - window_ids->item(oi) + 1; + cout << "i_start = " << i_start << endl; + cout << "Number of NNLS equations: " << NB * current_size << endl; + Gt.setSize(NQ, NB * current_size); + } + + // For every snapshot in batch + for (int i = i_start; i < current_size; ++i) + { + // Set the sampled dofs from the snapshot matrix + for (int j = 0; j < BX_snapshots->numRows(); ++j) + x_i[j] = (*BX_snapshots)(j, i); + + // Get prolongated dofs + Vector px_i; + Vector elfun; + + px_i.SetSize(P->Height()); + P->Mult(x_i, px_i); + + if (skipFirstW && i > 0 && i % nsnapPerSet == 0) + skip++; + + // For each basis vector + for (int j = 0; j < NB; ++j) + { + + // Get basis vector + for (int k = 0; k < BV->numRows(); ++k) + v_j[k] = (*BV)(k, j); + + // Get prolongated dofs + Vector pv_j; + Vector ve_j; + + pv_j.SetSize(P->Height()); + P->Mult(v_j, pv_j); + + // TODO: is it better to make the element loop the outer loop? + // For each element + for (int e = 0; e < ne; ++e) + { + // Get element and its dofs and transformation. + Array vdofs; + DofTransformation *doftrans = fespace_X->GetElementVDofs(e, vdofs); + const FiniteElement &fe = *fespace_X->GetFE(e); + ElementTransformation *eltrans = fespace_X->GetElementTransformation(e); + px_i.GetSubVector(vdofs, elfun); + pv_j.GetSubVector(vdofs, ve_j); + if (doftrans) + { + MFEM_ABORT("Doftrans is true, make corresponding edits") + } + + // Compute the row of G corresponding to element e, store in r + ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r); + + for (int m = 0; m < nqe; ++m) + Gt((e * nqe) + m, j + (i * NB)) = r[m]; + } + } + + if (precondition) + { + MFEM_ABORT("TODO"); + } + } // Loop (i) over snapshots + + Array const &w_el = ir0->GetWeights(); + MFEM_VERIFY(w_el.Size() == nqe, ""); + + CAROM::Vector w(ne * nqe, true); + + for (int i = 0; i < ne; ++i) + { + for (int j = 0; j < nqe; ++j) + w((i * nqe) + j) = w_el[j]; + } + + SolveNNLS(rank, nnls_tol, maxNNLSnnz, w, Gt, sol); + if (window_ids) + { + save_CAROM_Vector(sol, "sol_" + std::to_string(oi) + ".csv"); + } + } +} + +void WriteMeshEQP(ParMesh *pmesh, const int myid, const int nqe, + CAROM::Vector const &eqpSol) +{ + // Find the elements with quadrature points in eqpSol. + std::set elements; + Vector elemCount(pmesh->GetNE()); + elemCount = 0.0; + + for (int i = 0; i < eqpSol.dim(); ++i) + { + if (eqpSol(i) > 1.0e-12) + { + const int e = i / nqe; // Element index + elements.insert(e); + elemCount[e] += 1.0; + } + } + + // Empty sets, since EQP on samples inside elements. + std::set faces, edges, vertices; + CAROM::SampleVisualization(pmesh, elements, elements, faces, edges, + vertices, "EQPvis", &elemCount); +} + +// Function to compute the indices at which to switch time windows +void get_window_ids(int n_step, int n_window, CAROM::Vector *ids) +{ + int window_size = std::round(n_step / n_window); + + int res = n_step - (n_window * (window_size - 1) + 1); + int res_up = res - n_window; + int ctr = -1; + + for (int i = 1; i <= n_window + 1; ++i) + { + ids->item(i - 1) = (i - 1) * window_size + 1 - (i - 1); + if (res_up > 0) + { + if (i <= res - res_up) + { + ctr += 1; + } + if (i > 2 && i <= res_up + 1) + { + ctr += 1; + } + } + else + { + if (i <= res + 1) + { + ctr += 1; + } + } + ids->item(i - 1) += ctr; + } +} + +void load_CAROM_vector(const std::string &filename, CAROM::Vector &vector) +{ + std::ifstream file(filename); + std::string line; + std::vector data; + + while (std::getline(file, line)) + { + std::stringstream ss(line); + std::string value; + + while (std::getline(ss, value, ',')) + { + data.push_back(std::stod(value)); + } + } + + // Set the size of the vector + int size = data.size(); + vector.setSize(size); + + // Copy the data into the vector + for (int i = 0; i < size; ++i) + { + vector(i) = data[i]; + } +} + +void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol) +{ + string filename = "sol_" + std::to_string(current_window) + ".csv"; + if (fileExists(filename)) + { + load_CAROM_vector(filename, *load_eqpsol); + } +} From ec3cbe545d842478fa7740a49afdc9025c56c115 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:29:02 -0400 Subject: [PATCH 075/120] Fixed .hpp style issue --- .../prom/nonlinear_elasticity_global_rom.cpp | 770 +++++++++++++++++- .../nonlinear_elasticity_global_rom_eqp.cpp | 122 --- .../nonlinear_elasticity_global_rom_eqp.hpp | 634 +------------- 3 files changed, 787 insertions(+), 739 deletions(-) delete mode 100644 examples/prom/nonlinear_elasticity_global_rom_eqp.cpp diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 8c9296282..33e83e58a 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1945,4 +1945,772 @@ void RomOperator::SetEQP(CAROM::Vector *eqpSol) GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef); -} \ No newline at end of file +} + +// Functions for EQP functionality + +// Compute coefficients of the reduced integrator with respect to inputs Q and x +// in HyperelasticNLFIntegrator_ComputeReducedEQP. +void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, + CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef) +{ + const int rvdim = V_v.numColumns(); + const int fomdim = V_v.numRows(); + MFEM_VERIFY(rw.size() == qp.size(), ""); + MFEM_VERIFY(fomdim == fesR->GetTrueVSize(), ""); + + MFEM_VERIFY(rank == 0, + "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); + + const int nqe = ir->GetWeights().Size(); + + DofTransformation *doftrans; + ElementTransformation *eltrans; + Array vdofs; + + int eprev = -1; + int dof = 0; + int dim = 0; + + // Get prolongation matrix + const Operator *P = fesR->GetProlongationMatrix(); + + // Vector to be prolongated + Vector vj(fomdim); + + // Prolongated vector + Vector p_vj(P->Height()); + + // Element vector + Vector vj_e; + // Get the element vector size + doftrans = fesR->GetElementVDofs(0, vdofs); + int elvect_size = vdofs.Size(); + + // Coefficient vector + coef.SetSize(elvect_size * rw.size() * rvdim); + coef = 0.0; + + // Vector for storing DS + const FiniteElement *fe = fesR->GetFE(0); + dof = fe->GetDof(); + dim = fe->GetDim(); + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DS_coef.SetSize(dof * dim * rw.size() * rvdim); + DS_coef = 0.0; + int index = 0; + + // For every basis vector + for (int j = 0; j < rvdim; ++j) + { + // Get basis vector and prolongate + for (int k = 0; k < V_v.numRows(); ++k) + vj[k] = V_v(k, j); + P->Mult(vj, p_vj); + + eprev = -1; + + // For every quadrature weight + for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 + { + const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); + + if (e != eprev) // Update element transformation + { + doftrans = fesR->GetElementVDofs(e, vdofs); + eltrans = fesR->GetElementTransformation(e); + + if (doftrans) + { + MFEM_ABORT("TODO"); + } + + // Get element vectors + p_vj.GetSubVector(vdofs, vj_e); + eprev = e; + } + + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); + // Get the transformation weight + double t = eltrans->Weight(); + // Calculate r[i] = ve_j^T * elvect + for (int k = 0; k < elvect_size; k++) + { + coef[k + (i * elvect_size) + (j * rw.size() * elvect_size)] = vj_e[k] * rw[i] * t; + } + + // Calculate DS and store + CalcInverse(eltrans->Jacobian(), Jrt); + fe->CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + for (int ii = 0; ii < dof; ++ii) + { + for (int jj = 0; jj < dim; ++jj) + { + index = jj + ii * dim; + DS_coef[index + (i * dof*dim) + (j * rw.size() * dof*dim)] = DS.Elem(ii,jj); + } + } + } + } +} + +void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, + CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res) +{ + const int rxdim = V_x.numColumns(); + const int rvdim = V_v.numColumns(); + const int fomdim = V_v.numRows(); + MFEM_VERIFY(rw.size() == qp.size(), ""); + MFEM_VERIFY(x.dim() == rxdim, ""); + MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); + + MFEM_VERIFY(rank == 0, + "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); + + const int nqe = ir->GetWeights().Size(); + + ElementTransformation *eltrans; + DofTransformation *doftrans; + const FiniteElement *fe = NULL; + Array vdofs; + + res.SetSize(rxdim); + res = 0.0; + + int eprev = -1; + int dof = 0; + int dim = 0; + + // Get prolongation matrix + const Operator *P = fesR->GetProlongationMatrix(); + + // Vectors to be prolongated + CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); + Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); + Vector Vx(fomdim); + Vector vj(fomdim); + + // Prolongated vectors + Vector p_Vx(P->Height()); + Vector p_vj(P->Height()); + + // Element vectors + Vector Vx_e; + Vector vj_e; + + // Lift x, add x0 and prolongate result + V_x.mult(x, Vx_librom_temp); + add(*Vx_temp, *x0, Vx); + P->Mult(Vx, p_Vx); + + // Initialize nonlinear operator storage + // Assuming all elements have the same dim and n_dof + fe = fesR->GetFE(0); + dof = fe->GetDof(); + dim = fe->GetDim(); + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DenseMatrix Jpt(dim); + DenseMatrix P_f(dim); + DenseMatrix PMatI; // Extract element dofs + DenseMatrix PMatO; + Vector elvect(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); + + // For every basis vector + for (int j = 0; j < rvdim; ++j) + + { + // Get basis vector and prolongate + for (int k = 0; k < V_v.numRows(); ++k) + vj[k] = V_v(k, j); + P->Mult(vj, p_vj); + res[j] = 0.0; + + eprev = -1; + + // For every quadrature weight + for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 + { + const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); + + if (e != eprev) // Update element transformation + { + doftrans = fesR->GetElementVDofs(e, vdofs); + fe = fesR->GetFE(e); + eltrans = fesR->GetElementTransformation(e); + + dof = fe->GetDof(); // Get number of dofs in element + dim = fe->GetDim(); + + if (doftrans) + { + MFEM_ABORT("TODO"); + } + + // Get element vectors + p_Vx.GetSubVector(vdofs, Vx_e); + p_vj.GetSubVector(vdofs, vj_e); + eprev = e; + } + + // Integration at ip + PMatI.UseExternalData(Vx_e.GetData(), dof, dim); + + elvect = 0.0; + + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); + + // Get the transformation weight + double t = eltrans->Weight(); + + // Compute action of nonlinear operator + CalcInverse(eltrans->Jacobian(), Jrt); + fe->CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P_f); + P_f *= (t * rw[i]); // NB: Not by ip.weight + AddMultABt(DS, P_f, PMatO); + + // Calculate r[i] = ve_j^T * elvect + for (int k = 0; k < elvect.Size(); k++) + { + res[j] += vj_e[k] * elvect[k]; + } + } + } +} + + +void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, + std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, + const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, + Vector const &coef, Vector const &DS_coef, const int rank, Vector &res) +{ + + const int rxdim = V_x.numColumns(); + const int rvdim = V_v.numColumns(); + const int fomdim = V_x.numRows(); + MFEM_VERIFY(x.dim() == rxdim, ""); + MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); + + MFEM_VERIFY(rank == 0, + "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); + + const int nqe = ir->GetWeights().Size(); + + ElementTransformation *eltrans; + DofTransformation *doftrans; + const FiniteElement *fe = NULL; + Array vdofs; + + res.SetSize(rxdim); + res = 0.0; + + int eprev = -1; + int dof = 0; + int dim = 0; + + // Get prolongation matrix + const Operator *P = fesR->GetProlongationMatrix(); + + // Vectors to be prolongated + CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); + Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); + Vector Vx(fomdim); + + // Prolongated vectors + Vector p_Vx(P->Height()); + + // Element vectors + Vector Vx_e; + + // Lift x, add x0 and prolongate result + V_x.mult(x, Vx_librom_temp); + add(*Vx_temp, *x0, Vx); + P->Mult(Vx, p_Vx); + + // Initialize nonlinear operator storage + // Assuming all elements have the same dim and n_dof + fe = fesR->GetFE(0); + dof = fe->GetDof(); + dim = fe->GetDim(); + int index = 0; + DenseMatrix DS(dof, dim); + DenseMatrix Jpt(dim); + DenseMatrix P_f(dim); + DenseMatrix PMatI; // Extract element dofs + DenseMatrix PMatO; + Vector elvect(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); + + // For every basis vector + for (int j = 0; j < rvdim; ++j) + + { + + eprev = -1; + + // For every quadrature weight + for (int i = 0; i < qp.size(); ++i) // NOTE: i < 9 + { + const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); + + if (e != eprev) // Update element transformation + { + doftrans = fesR->GetElementVDofs(e, vdofs); + fe = fesR->GetFE(e); + eltrans = fesR->GetElementTransformation(e); + + dof = fe->GetDof(); // Get number of dofs in element + dim = fe->GetDim(); + + if (doftrans) + { + MFEM_ABORT("TODO"); + } + + // Get element vectors + p_Vx.GetSubVector(vdofs, Vx_e); + eprev = e; + } + + // Integration at ip + elvect = 0.0; + PMatI.UseExternalData(Vx_e.GetData(), dof, dim); + + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); + + for (int ii = 0; ii < dof; ++ii) + { + for (int jj = 0; jj < dim; ++jj) + { + index = jj + ii * dim; + DS.Elem(ii,jj) = DS_coef[index + (i * elvect.Size()) + (j * qp.size() * elvect.Size())]; + } + } + + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P_f); + AddMultABt(DS, P_f, PMatO); + + // Calculate r[i] = ve_j^T * elvect + // coef is size len(vdofs) * rvdim * rw.size + for (int k = 0; k < elvect.Size(); k++) + { + res[j] += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * elvect[k]; + } + } + } +} + +void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, + Vector const &ve_j, NeoHookeanModel *model, Vector const &elfun, + FiniteElement const &fe, ElementTransformation &Trans, Vector &r) +{ + MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); + int dof = fe.GetDof(); // Get number of dofs in element + int dim = fe.GetDim(); + + // Initialize nonlinear operator matrices (there is probably a better way) + DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DenseMatrix Jpt(dim); + DenseMatrix P(dim); + DenseMatrix PMatI; // Extract element dofs + PMatI.UseExternalData(elfun.GetData(), dof, dim); + DenseMatrix PMatO; + Vector elvect(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); + + model->SetTransformation(Trans); + + // For each integration point + for (int i = 0; i < ir->GetNPoints(); i++) + { + elvect = 0.0; + // Get integration point + const IntegrationPoint &ip = ir->IntPoint(i); + + // Set integration point in the element transformation + Trans.SetIntPoint(&ip); + + // Get the transformation weight + double t = Trans.Weight(); + + // Compute action of nonlinear operator + CalcInverse(Trans.Jacobian(), Jrt); + fe.CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P); + P *= t; // NB: Not by ip.weight + AddMultABt(DS, P, PMatO); + + r[i] = 0.0; + + // Calculate r[i] = ve_j^T * elvect + for (int k = 0; k < elvect.Size(); k++) + { + r[i] += ve_j[k] * elvect[k]; + } + } +} + + +void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, + CAROM::Vector const &w, CAROM::Matrix &Gt, + CAROM::Vector &sol) +{ + CAROM::NNLSSolver nnls(nnls_tol, 0, maxNNLSnnz, 2); + + CAROM::Vector rhs_ub(Gt.numColumns(), false); + // G.mult(w, rhs_ub); // rhs = Gw + // rhs = Gw. Note that by using Gt and multTranspose, we do parallel communication. + Gt.transposeMult(w, rhs_ub); + + CAROM::Vector rhs_lb(rhs_ub); + CAROM::Vector rhs_Gw(rhs_ub); + + const double delta = 1.0e-11; + for (int i = 0; i < rhs_ub.dim(); ++i) + { + rhs_lb(i) -= delta; + rhs_ub(i) += delta; + } + + // nnls.normalize_constraints(Gt, rhs_lb, rhs_ub); + nnls.solve_parallel_with_scalapack(Gt, rhs_lb, rhs_ub, sol); + + int nnz = 0; + for (int i = 0; i < sol.dim(); ++i) + { + if (sol(i) != 0.0) + { + nnz++; + } + } + + cout << rank << ": Number of nonzeros in NNLS solution: " << nnz + << ", out of " << sol.dim() << endl; + + MPI_Allreduce(MPI_IN_PLACE, &nnz, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + + if (rank == 0) + cout << "Global number of nonzeros in NNLS solution: " << nnz << endl; + + // Check residual of NNLS solution + CAROM::Vector res(Gt.numColumns(), false); + Gt.transposeMult(sol, res); + + const double normGsol = res.norm(); + const double normRHS = rhs_Gw.norm(); + + res -= rhs_Gw; + const double relNorm = res.norm() / std::max(normGsol, normRHS); + cout << rank << ": relative residual norm for NNLS solution of Gs = Gw: " << relNorm << endl; +} + + +// Compute EQP solution from constraints on snapshots. +void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, + ParFiniteElementSpace *fespace_X, + const int nsets, const CAROM::Matrix *BV, + const CAROM::Matrix *BX_snapshots, + const bool precondition, const double nnls_tol, + const int maxNNLSnnz, NeoHookeanModel *model, + CAROM::Vector &sol, CAROM::Vector *window_ids) +{ + const int nqe = ir0->GetNPoints(); + const int ne = fespace_X->GetNE(); + const int NB = BV->numColumns(); + const int NQ = ne * nqe; + const int nsnap = BX_snapshots->numColumns(); + + MFEM_VERIFY(nsnap == BX_snapshots->numColumns() || + nsnap + nsets == BX_snapshots->numColumns(), // Q: nsets? + ""); + MFEM_VERIFY(BV->numRows() == BX_snapshots->numRows(), ""); + MFEM_VERIFY(BV->numRows() == fespace_X->GetTrueVSize(), ""); + + const bool skipFirstW = (nsnap + nsets == BX_snapshots->numColumns()); + + // Compute G of size (NB * nsnap) x NQ, but only store its transpose Gt. + CAROM::Matrix Gt(NQ, NB * nsnap, true); + int current_size = 0; + int i_start = 0; + int outer_loop_length = 0; + if (!window_ids) + { + current_size = nsnap; + outer_loop_length = 1; + } + else + { + outer_loop_length = window_ids->dim() - 1; + } + + // For 0 <= j < NB, 0 <= i < nsnap, 0 <= e < ne, 0 <= m < nqe, + // G(j + (i*NB), (e*nqe) + m) + // is the coefficient of v_j^T M(p_i) V v_i at point m of element e, + // with respect to the integration rule weight at that point, + // where the "exact" quadrature solution is ir0->GetWeights(). + + Vector x_i(BX_snapshots->numRows()); + Vector v_j(BV->numRows()); + + Vector r(nqe); + + int skip = 0; + const int nsnapPerSet = nsnap / nsets; + if (skipFirstW) + { + MFEM_VERIFY(nsets * nsnapPerSet == nsnap, ""); + skip = 1; + } + + // Get prolongation matrix + const Operator *P = fespace_X->GetProlongationMatrix(); + if (!P) + { + MFEM_ABORT("P is null, generalize to serial case") + } + + // Outer loop for time windowing + for (int oi = 0; oi < outer_loop_length; ++oi) + { + if (window_ids) + { + i_start = window_ids->item(oi); + current_size = window_ids->item(oi + 1) - window_ids->item(oi) + 1; + cout << "i_start = " << i_start << endl; + cout << "Number of NNLS equations: " << NB * current_size << endl; + Gt.setSize(NQ, NB * current_size); + } + + // For every snapshot in batch + for (int i = i_start; i < current_size; ++i) + { + // Set the sampled dofs from the snapshot matrix + for (int j = 0; j < BX_snapshots->numRows(); ++j) + x_i[j] = (*BX_snapshots)(j, i); + + // Get prolongated dofs + Vector px_i; + Vector elfun; + + px_i.SetSize(P->Height()); + P->Mult(x_i, px_i); + + if (skipFirstW && i > 0 && i % nsnapPerSet == 0) + skip++; + + // For each basis vector + for (int j = 0; j < NB; ++j) + { + + // Get basis vector + for (int k = 0; k < BV->numRows(); ++k) + v_j[k] = (*BV)(k, j); + + // Get prolongated dofs + Vector pv_j; + Vector ve_j; + + pv_j.SetSize(P->Height()); + P->Mult(v_j, pv_j); + + // TODO: is it better to make the element loop the outer loop? + // For each element + for (int e = 0; e < ne; ++e) + { + // Get element and its dofs and transformation. + Array vdofs; + DofTransformation *doftrans = fespace_X->GetElementVDofs(e, vdofs); + const FiniteElement &fe = *fespace_X->GetFE(e); + ElementTransformation *eltrans = fespace_X->GetElementTransformation(e); + px_i.GetSubVector(vdofs, elfun); + pv_j.GetSubVector(vdofs, ve_j); + if (doftrans) + { + MFEM_ABORT("Doftrans is true, make corresponding edits") + } + + // Compute the row of G corresponding to element e, store in r + ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r); + + for (int m = 0; m < nqe; ++m) + Gt((e * nqe) + m, j + (i * NB)) = r[m]; + } + } + + if (precondition) + { + MFEM_ABORT("TODO"); + } + } // Loop (i) over snapshots + + Array const &w_el = ir0->GetWeights(); + MFEM_VERIFY(w_el.Size() == nqe, ""); + + CAROM::Vector w(ne * nqe, true); + + for (int i = 0; i < ne; ++i) + { + for (int j = 0; j < nqe; ++j) + w((i * nqe) + j) = w_el[j]; + } + + SolveNNLS(rank, nnls_tol, maxNNLSnnz, w, Gt, sol); + if (window_ids) + { + save_CAROM_Vector(sol, "sol_" + std::to_string(oi) + ".csv"); + } + } +} + +// Utility functions +void WriteMeshEQP(ParMesh *pmesh, const int myid, const int nqe, + CAROM::Vector const &eqpSol) +{ + // Find the elements with quadrature points in eqpSol. + std::set elements; + Vector elemCount(pmesh->GetNE()); + elemCount = 0.0; + + for (int i = 0; i < eqpSol.dim(); ++i) + { + if (eqpSol(i) > 1.0e-12) + { + const int e = i / nqe; // Element index + elements.insert(e); + elemCount[e] += 1.0; + } + } + + // Empty sets, since EQP on samples inside elements. + std::set faces, edges, vertices; + CAROM::SampleVisualization(pmesh, elements, elements, faces, edges, + vertices, "EQPvis", &elemCount); +} + +// Function to compute the indices at which to switch time windows +void get_window_ids(int n_step, int n_window, CAROM::Vector *ids) +{ + int window_size = std::round(n_step / n_window); + + int res = n_step - (n_window * (window_size - 1) + 1); + int res_up = res - n_window; + int ctr = -1; + + for (int i = 1; i <= n_window + 1; ++i) + { + ids->item(i - 1) = (i - 1) * window_size + 1 - (i - 1); + if (res_up > 0) + { + if (i <= res - res_up) + { + ctr += 1; + } + if (i > 2 && i <= res_up + 1) + { + ctr += 1; + } + } + else + { + if (i <= res + 1) + { + ctr += 1; + } + } + ids->item(i - 1) += ctr; + } +} + + +bool fileExists(const std::string &filename) +{ + std::ifstream file(filename); + return file.good(); +} + +void save_CAROM_Vector(const CAROM::Vector &vector, const std::string &filename) +{ + std::ofstream file(filename); + if (file.is_open()) + { + for (int i = 0; i < vector.dim(); ++i) + { + file << vector(i) << "\n"; + } + file.close(); + std::cout << "Vector saved as " << filename << " successfully." << std::endl; + } + else + { + std::cerr << "Error: Unable to open file " << filename << " for writing." << std::endl; + } +} + +void load_CAROM_vector(const std::string &filename, CAROM::Vector &vector) +{ + std::ifstream file(filename); + std::string line; + std::vector data; + + while (std::getline(file, line)) + { + std::stringstream ss(line); + std::string value; + + while (std::getline(ss, value, ',')) + { + data.push_back(std::stod(value)); + } + } + + // Set the size of the vector + int size = data.size(); + vector.setSize(size); + + // Copy the data into the vector + for (int i = 0; i < size; ++i) + { + vector(i) = data[i]; + } +} + +void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol) +{ + string filename = "sol_" + std::to_string(current_window) + ".csv"; + if (fileExists(filename)) + { + load_CAROM_vector(filename, *load_eqpsol); + } +} diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.cpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.cpp deleted file mode 100644 index f05788cf3..000000000 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.cpp +++ /dev/null @@ -1,122 +0,0 @@ -#include "mfem/Utilities.hpp" -// #include "fem/nonlininteg.hpp" -#include -using namespace mfem; -using namespace std; - -#include "nonlinear_elasticity_global_rom_eqp.hpp" - -// Compute coefficients of the reduced integrator with respect to inputs Q and x -// in HyperelasticNLFIntegrator_ComputeReducedEQP. -void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, - std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, - CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef) -{ - const int rvdim = V_v.numColumns(); - const int fomdim = V_v.numRows(); - MFEM_VERIFY(rw.size() == qp.size(), ""); - MFEM_VERIFY(fomdim == fesR->GetTrueVSize(), ""); - - MFEM_VERIFY(rank == 0, - "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); - - const int nqe = ir->GetWeights().Size(); - - DofTransformation *doftrans; - ElementTransformation *eltrans; - Array vdofs; - - int eprev = -1; - int dof = 0; - int dim = 0; - - // Get prolongation matrix - const Operator *P = fesR->GetProlongationMatrix(); - - // Vector to be prolongated - Vector vj(fomdim); - - // Prolongated vector - Vector p_vj(P->Height()); - - // Element vector - Vector vj_e; - // Get the element vector size - doftrans = fesR->GetElementVDofs(0, vdofs); - int elvect_size = vdofs.Size(); - - // Coefficient vector - coef.SetSize(elvect_size * rw.size() * rvdim); - coef = 0.0; - - // Vector for storing DS - const FiniteElement *fe = fesR->GetFE(0); - dof = fe->GetDof(); - dim = fe->GetDim(); - DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DS_coef.SetSize(dof * dim * rw.size() * rvdim); - DS_coef = 0.0; - int index = 0; - - // For every basis vector - for (int j = 0; j < rvdim; ++j) - { - // Get basis vector and prolongate - for (int k = 0; k < V_v.numRows(); ++k) - vj[k] = V_v(k, j); - P->Mult(vj, p_vj); - - eprev = -1; - - // For every quadrature weight - for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 - { - const int e = qp[i] / nqe; // Element index - // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); - const IntegrationPoint &ip = ir->IntPoint(qpi); - - if (e != eprev) // Update element transformation - { - doftrans = fesR->GetElementVDofs(e, vdofs); - eltrans = fesR->GetElementTransformation(e); - - if (doftrans) - { - MFEM_ABORT("TODO"); - } - - // Get element vectors - p_vj.GetSubVector(vdofs, vj_e); - eprev = e; - } - - // Set integration point in the element transformation - eltrans->SetIntPoint(&ip); - model->SetTransformation(*eltrans); - // Get the transformation weight - double t = eltrans->Weight(); - // Calculate r[i] = ve_j^T * elvect - for (int k = 0; k < elvect_size; k++) - { - coef[k + (i * elvect_size) + (j * rw.size() * elvect_size)] = vj_e[k] * rw[i] * t; - } - - // Calculate DS and store - CalcInverse(eltrans->Jacobian(), Jrt); - fe->CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - for (int ii = 0; ii < dof; ++ii) - { - for (int jj = 0; jj < dim; ++jj) - { - index = jj + ii * dim; - DS_coef[index + (i * dof*dim) + (j * rw.size() * dof*dim)] = DS.Elem(ii,jj); - } - } - } - } -} diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 63cf63d67..da7b6b7f3 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -59,399 +59,26 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, const IntegrationRule *ir, NeoHookeanModel *model, CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef); +// Perform hyperreduction with EQP void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, - CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res) -{ - const int rxdim = V_x.numColumns(); - const int rvdim = V_v.numColumns(); - const int fomdim = V_v.numRows(); - MFEM_VERIFY(rw.size() == qp.size(), ""); - MFEM_VERIFY(x.dim() == rxdim, ""); - MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); - - MFEM_VERIFY(rank == 0, - "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); - - const int nqe = ir->GetWeights().Size(); - - ElementTransformation *eltrans; - DofTransformation *doftrans; - const FiniteElement *fe = NULL; - Array vdofs; - - res.SetSize(rxdim); - res = 0.0; - - int eprev = -1; - int dof = 0; - int dim = 0; - - // Get prolongation matrix - const Operator *P = fesR->GetProlongationMatrix(); - - // Vectors to be prolongated - CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); - Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); - Vector Vx(fomdim); - Vector vj(fomdim); - - // Prolongated vectors - Vector p_Vx(P->Height()); - Vector p_vj(P->Height()); - - // Element vectors - Vector Vx_e; - Vector vj_e; - - // Lift x, add x0 and prolongate result - V_x.mult(x, Vx_librom_temp); - add(*Vx_temp, *x0, Vx); - P->Mult(Vx, p_Vx); - - // Initialize nonlinear operator storage - // Assuming all elements have the same dim and n_dof - fe = fesR->GetFE(0); - dof = fe->GetDof(); - dim = fe->GetDim(); - DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DenseMatrix Jpt(dim); - DenseMatrix P_f(dim); - DenseMatrix PMatI; // Extract element dofs - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); - - // For every basis vector - for (int j = 0; j < rvdim; ++j) - - { - // Get basis vector and prolongate - for (int k = 0; k < V_v.numRows(); ++k) - vj[k] = V_v(k, j); - P->Mult(vj, p_vj); - res[j] = 0.0; - - eprev = -1; - - // For every quadrature weight - for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 - { - const int e = qp[i] / nqe; // Element index - // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); - const IntegrationPoint &ip = ir->IntPoint(qpi); - - if (e != eprev) // Update element transformation - { - doftrans = fesR->GetElementVDofs(e, vdofs); - fe = fesR->GetFE(e); - eltrans = fesR->GetElementTransformation(e); - - dof = fe->GetDof(); // Get number of dofs in element - dim = fe->GetDim(); - - if (doftrans) - { - MFEM_ABORT("TODO"); - } - - // Get element vectors - p_Vx.GetSubVector(vdofs, Vx_e); - p_vj.GetSubVector(vdofs, vj_e); - eprev = e; - } - - // Integration at ip - PMatI.UseExternalData(Vx_e.GetData(), dof, dim); - - elvect = 0.0; - - // Set integration point in the element transformation - eltrans->SetIntPoint(&ip); - model->SetTransformation(*eltrans); - - // Get the transformation weight - double t = eltrans->Weight(); - - // Compute action of nonlinear operator - CalcInverse(eltrans->Jacobian(), Jrt); - fe->CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P_f); - P_f *= (t * rw[i]); // NB: Not by ip.weight - AddMultABt(DS, P_f, PMatO); - - // Calculate r[i] = ve_j^T * elvect - for (int k = 0; k < elvect.Size(); k++) - { - res[j] += vj_e[k] * elvect[k]; - } - } - } -} + CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res); +// Optimized EQP hyperreduction routine with preallocated arrays void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, - Vector const &coef, Vector const &DS_coef, const int rank, Vector &res) -{ - - const int rxdim = V_x.numColumns(); - const int rvdim = V_v.numColumns(); - const int fomdim = V_x.numRows(); - MFEM_VERIFY(x.dim() == rxdim, ""); - MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); - - MFEM_VERIFY(rank == 0, - "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); - - const int nqe = ir->GetWeights().Size(); - - ElementTransformation *eltrans; - DofTransformation *doftrans; - const FiniteElement *fe = NULL; - Array vdofs; - - res.SetSize(rxdim); - res = 0.0; - - int eprev = -1; - int dof = 0; - int dim = 0; - - // Get prolongation matrix - const Operator *P = fesR->GetProlongationMatrix(); - - // Vectors to be prolongated - CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); - Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); - Vector Vx(fomdim); - - // Prolongated vectors - Vector p_Vx(P->Height()); - - // Element vectors - Vector Vx_e; - - // Lift x, add x0 and prolongate result - V_x.mult(x, Vx_librom_temp); - add(*Vx_temp, *x0, Vx); - P->Mult(Vx, p_Vx); - - // Initialize nonlinear operator storage - // Assuming all elements have the same dim and n_dof - fe = fesR->GetFE(0); - dof = fe->GetDof(); - dim = fe->GetDim(); - int index = 0; - DenseMatrix DS(dof, dim); - DenseMatrix Jpt(dim); - DenseMatrix P_f(dim); - DenseMatrix PMatI; // Extract element dofs - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); - - // For every basis vector - for (int j = 0; j < rvdim; ++j) - - { - - eprev = -1; - - // For every quadrature weight - for (int i = 0; i < qp.size(); ++i) // NOTE: i < 9 - { - const int e = qp[i] / nqe; // Element index - // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); - const IntegrationPoint &ip = ir->IntPoint(qpi); - - if (e != eprev) // Update element transformation - { - doftrans = fesR->GetElementVDofs(e, vdofs); - fe = fesR->GetFE(e); - eltrans = fesR->GetElementTransformation(e); - - dof = fe->GetDof(); // Get number of dofs in element - dim = fe->GetDim(); - - if (doftrans) - { - MFEM_ABORT("TODO"); - } - - // Get element vectors - p_Vx.GetSubVector(vdofs, Vx_e); - eprev = e; - } - - // Integration at ip - elvect = 0.0; - PMatI.UseExternalData(Vx_e.GetData(), dof, dim); - - // Set integration point in the element transformation - eltrans->SetIntPoint(&ip); - model->SetTransformation(*eltrans); - - for (int ii = 0; ii < dof; ++ii) - { - for (int jj = 0; jj < dim; ++jj) - { - index = jj + ii * dim; - DS.Elem(ii,jj) = DS_coef[index + (i * elvect.Size()) + (j * qp.size() * elvect.Size())]; - } - } - - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P_f); - AddMultABt(DS, P_f, PMatO); - - // Calculate r[i] = ve_j^T * elvect - // coef is size len(vdofs) * rvdim * rw.size - for (int k = 0; k < elvect.Size(); k++) - { - res[j] += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * elvect[k]; - } - } - } -} - -bool fileExists(const std::string &filename) -{ - std::ifstream file(filename); - return file.good(); -} - -void save_CAROM_Vector(const CAROM::Vector &vector, const std::string &filename) -{ - std::ofstream file(filename); - if (file.is_open()) - { - for (int i = 0; i < vector.dim(); ++i) - { - file << vector(i) << "\n"; - } - file.close(); - std::cout << "Vector saved as " << filename << " successfully." << std::endl; - } - else - { - std::cerr << "Error: Unable to open file " << filename << " for writing." << std::endl; - } -} + Vector const &coef, Vector const &DS_coef, const int rank, Vector &res); +// Compute a row in the G matrix which corresponds to a given FE element void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &ve_j, NeoHookeanModel *model, Vector const &elfun, - FiniteElement const &fe, ElementTransformation &Trans, Vector &r) -{ - MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); - int dof = fe.GetDof(); // Get number of dofs in element - int dim = fe.GetDim(); - - // Initialize nonlinear operator matrices (there is probably a better way) - DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DenseMatrix Jpt(dim); - DenseMatrix P(dim); - DenseMatrix PMatI; // Extract element dofs - PMatI.UseExternalData(elfun.GetData(), dof, dim); - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); - - model->SetTransformation(Trans); - - // For each integration point - for (int i = 0; i < ir->GetNPoints(); i++) - { - elvect = 0.0; - // Get integration point - const IntegrationPoint &ip = ir->IntPoint(i); - - // Set integration point in the element transformation - Trans.SetIntPoint(&ip); - - // Get the transformation weight - double t = Trans.Weight(); - - // Compute action of nonlinear operator - CalcInverse(Trans.Jacobian(), Jrt); - fe.CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); - P *= t; // NB: Not by ip.weight - AddMultABt(DS, P, PMatO); - - r[i] = 0.0; - - // Calculate r[i] = ve_j^T * elvect - for (int k = 0; k < elvect.Size(); k++) - { - r[i] += ve_j[k] * elvect[k]; - } - } -} + FiniteElement const &fe, ElementTransformation &Trans, Vector &r); void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, CAROM::Vector const &w, CAROM::Matrix &Gt, - CAROM::Vector &sol) -{ - CAROM::NNLSSolver nnls(nnls_tol, 0, maxNNLSnnz, 2); - - CAROM::Vector rhs_ub(Gt.numColumns(), false); - // G.mult(w, rhs_ub); // rhs = Gw - // rhs = Gw. Note that by using Gt and multTranspose, we do parallel communication. - Gt.transposeMult(w, rhs_ub); - - CAROM::Vector rhs_lb(rhs_ub); - CAROM::Vector rhs_Gw(rhs_ub); - - const double delta = 1.0e-11; - for (int i = 0; i < rhs_ub.dim(); ++i) - { - rhs_lb(i) -= delta; - rhs_ub(i) += delta; - } - - // nnls.normalize_constraints(Gt, rhs_lb, rhs_ub); - nnls.solve_parallel_with_scalapack(Gt, rhs_lb, rhs_ub, sol); - - int nnz = 0; - for (int i = 0; i < sol.dim(); ++i) - { - if (sol(i) != 0.0) - { - nnz++; - } - } - - cout << rank << ": Number of nonzeros in NNLS solution: " << nnz - << ", out of " << sol.dim() << endl; - - MPI_Allreduce(MPI_IN_PLACE, &nnz, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - - if (rank == 0) - cout << "Global number of nonzeros in NNLS solution: " << nnz << endl; - - // Check residual of NNLS solution - CAROM::Vector res(Gt.numColumns(), false); - Gt.transposeMult(sol, res); - - const double normGsol = res.norm(); - const double normRHS = rhs_Gw.norm(); - - res -= rhs_Gw; - const double relNorm = res.norm() / std::max(normGsol, normRHS); - cout << rank << ": relative residual norm for NNLS solution of Gs = Gw: " << relNorm << endl; -} + CAROM::Vector &sol); // Compute EQP solution from constraints on snapshots. void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, @@ -460,247 +87,22 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const CAROM::Matrix *BX_snapshots, const bool precondition, const double nnls_tol, const int maxNNLSnnz, NeoHookeanModel *model, - CAROM::Vector &sol, CAROM::Vector *window_ids) -{ - const int nqe = ir0->GetNPoints(); - const int ne = fespace_X->GetNE(); - const int NB = BV->numColumns(); - const int NQ = ne * nqe; - const int nsnap = BX_snapshots->numColumns(); - - MFEM_VERIFY(nsnap == BX_snapshots->numColumns() || - nsnap + nsets == BX_snapshots->numColumns(), // Q: nsets? - ""); - MFEM_VERIFY(BV->numRows() == BX_snapshots->numRows(), ""); - MFEM_VERIFY(BV->numRows() == fespace_X->GetTrueVSize(), ""); - - const bool skipFirstW = (nsnap + nsets == BX_snapshots->numColumns()); - - // Compute G of size (NB * nsnap) x NQ, but only store its transpose Gt. - CAROM::Matrix Gt(NQ, NB * nsnap, true); - int current_size = 0; - int i_start = 0; - int outer_loop_length = 0; - if (!window_ids) - { - current_size = nsnap; - outer_loop_length = 1; - } - else - { - outer_loop_length = window_ids->dim() - 1; - } - - // For 0 <= j < NB, 0 <= i < nsnap, 0 <= e < ne, 0 <= m < nqe, - // G(j + (i*NB), (e*nqe) + m) - // is the coefficient of v_j^T M(p_i) V v_i at point m of element e, - // with respect to the integration rule weight at that point, - // where the "exact" quadrature solution is ir0->GetWeights(). - - Vector x_i(BX_snapshots->numRows()); - Vector v_j(BV->numRows()); - - Vector r(nqe); - - int skip = 0; - const int nsnapPerSet = nsnap / nsets; - if (skipFirstW) - { - MFEM_VERIFY(nsets * nsnapPerSet == nsnap, ""); - skip = 1; - } - - // Get prolongation matrix - const Operator *P = fespace_X->GetProlongationMatrix(); - if (!P) - { - MFEM_ABORT("P is null, generalize to serial case") - } - - // Outer loop for time windowing - for (int oi = 0; oi < outer_loop_length; ++oi) - { - if (window_ids) - { - i_start = window_ids->item(oi); - current_size = window_ids->item(oi + 1) - window_ids->item(oi) + 1; - cout << "i_start = " << i_start << endl; - cout << "Number of NNLS equations: " << NB * current_size << endl; - Gt.setSize(NQ, NB * current_size); - } - - // For every snapshot in batch - for (int i = i_start; i < current_size; ++i) - { - // Set the sampled dofs from the snapshot matrix - for (int j = 0; j < BX_snapshots->numRows(); ++j) - x_i[j] = (*BX_snapshots)(j, i); - - // Get prolongated dofs - Vector px_i; - Vector elfun; - - px_i.SetSize(P->Height()); - P->Mult(x_i, px_i); - - if (skipFirstW && i > 0 && i % nsnapPerSet == 0) - skip++; - - // For each basis vector - for (int j = 0; j < NB; ++j) - { - - // Get basis vector - for (int k = 0; k < BV->numRows(); ++k) - v_j[k] = (*BV)(k, j); - - // Get prolongated dofs - Vector pv_j; - Vector ve_j; - - pv_j.SetSize(P->Height()); - P->Mult(v_j, pv_j); - - // TODO: is it better to make the element loop the outer loop? - // For each element - for (int e = 0; e < ne; ++e) - { - // Get element and its dofs and transformation. - Array vdofs; - DofTransformation *doftrans = fespace_X->GetElementVDofs(e, vdofs); - const FiniteElement &fe = *fespace_X->GetFE(e); - ElementTransformation *eltrans = fespace_X->GetElementTransformation(e); - px_i.GetSubVector(vdofs, elfun); - pv_j.GetSubVector(vdofs, ve_j); - if (doftrans) - { - MFEM_ABORT("Doftrans is true, make corresponding edits") - } - - // Compute the row of G corresponding to element e, store in r - ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r); - - for (int m = 0; m < nqe; ++m) - Gt((e * nqe) + m, j + (i * NB)) = r[m]; - } - } - - if (precondition) - { - MFEM_ABORT("TODO"); - } - } // Loop (i) over snapshots - - Array const &w_el = ir0->GetWeights(); - MFEM_VERIFY(w_el.Size() == nqe, ""); - - CAROM::Vector w(ne * nqe, true); - - for (int i = 0; i < ne; ++i) - { - for (int j = 0; j < nqe; ++j) - w((i * nqe) + j) = w_el[j]; - } - - SolveNNLS(rank, nnls_tol, maxNNLSnnz, w, Gt, sol); - if (window_ids) - { - save_CAROM_Vector(sol, "sol_" + std::to_string(oi) + ".csv"); - } - } -} + CAROM::Vector &sol, CAROM::Vector *window_ids); void WriteMeshEQP(ParMesh *pmesh, const int myid, const int nqe, - CAROM::Vector const &eqpSol) -{ - // Find the elements with quadrature points in eqpSol. - std::set elements; - Vector elemCount(pmesh->GetNE()); - elemCount = 0.0; - - for (int i = 0; i < eqpSol.dim(); ++i) - { - if (eqpSol(i) > 1.0e-12) - { - const int e = i / nqe; // Element index - elements.insert(e); - elemCount[e] += 1.0; - } - } - - // Empty sets, since EQP on samples inside elements. - std::set faces, edges, vertices; - CAROM::SampleVisualization(pmesh, elements, elements, faces, edges, - vertices, "EQPvis", &elemCount); -} + CAROM::Vector const &eqpSol); // Function to compute the indices at which to switch time windows -void get_window_ids(int n_step, int n_window, CAROM::Vector *ids) -{ - int window_size = std::round(n_step / n_window); - - int res = n_step - (n_window * (window_size - 1) + 1); - int res_up = res - n_window; - int ctr = -1; +void get_window_ids(int n_step, int n_window, CAROM::Vector *ids); - for (int i = 1; i <= n_window + 1; ++i) - { - ids->item(i - 1) = (i - 1) * window_size + 1 - (i - 1); - if (res_up > 0) - { - if (i <= res - res_up) - { - ctr += 1; - } - if (i > 2 && i <= res_up + 1) - { - ctr += 1; - } - } - else - { - if (i <= res + 1) - { - ctr += 1; - } - } - ids->item(i - 1) += ctr; - } -} - -void load_CAROM_vector(const std::string &filename, CAROM::Vector &vector) -{ - std::ifstream file(filename); - std::string line; - std::vector data; +// Helper function to check if a file exists +bool fileExists(const std::string &filename); - while (std::getline(file, line)) - { - std::stringstream ss(line); - std::string value; +// Helper function to save a CAROM vector to an external file +void save_CAROM_Vector(const CAROM::Vector &vector, const std::string &filename); - while (std::getline(ss, value, ',')) - { - data.push_back(std::stod(value)); - } - } +// Helper function to load a CAROM vector from an external file +void load_CAROM_vector(const std::string &filename, CAROM::Vector &vector); - // Set the size of the vector - int size = data.size(); - vector.setSize(size); - - // Copy the data into the vector - for (int i = 0; i < size; ++i) - { - vector(i) = data[i]; - } -} - -void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol) -{ - string filename = "sol_" + std::to_string(current_window) + ".csv"; - if (fileExists(filename)) - { - load_CAROM_vector(filename, *load_eqpsol); - } -} +// Load a EQP solution +void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol); From e5f8556607a70d90cdc12899bdc35599e9e23c64 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:04:12 -0400 Subject: [PATCH 076/120] Delete the RomOperator to remove a memory leak --- examples/prom/nonlinear_elasticity_global_rom.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 33e83e58a..b00a89073 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1432,6 +1432,7 @@ int main(int argc, char *argv[]) // 16. Free the used memory. delete ode_solver; delete pmesh; + delete romop; totalTimer.Stop(); if (myid == 0) From 6faddd3428f1ab7a832e17c299999c213ef8fe03 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:32:30 -0400 Subject: [PATCH 077/120] Updated RomOperator structure to have less leaks in timestepping --- .../prom/nonlinear_elasticity_global_rom.cpp | 38 +++++++++---------- .../nonlinear_elasticity_global_rom_eqp.hpp | 7 ++-- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index b00a89073..245e6ff4f 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -161,9 +161,11 @@ class RomOperator : public TimeDependentOperator CAROM::Matrix *V_xTV_v_sp; CAROM::Matrix *V_v_sp_dist; CAROM::Vector *psp_librom, *psp_v_librom; + CAROM::Vector *Vx_librom_temp; Vector *psp; Vector *psp_x; Vector *psp_v; + Vector *Vx_temp; mutable Vector zH; mutable CAROM::Vector zX; mutable Vector zX_MFEM; @@ -312,8 +314,8 @@ void BasisGeneratorFinalSummary(CAROM::BasisGenerator* bg, double partialSum = 0.0; stringstream prec; - int ctr = 1; - char buffer[100]; + int ctr = 1; + char buffer[100]; for (int sv = 0; sv < sing_vals->dim(); ++sv) { partialSum += (*sing_vals)(sv); @@ -324,7 +326,7 @@ void BasisGeneratorFinalSummary(CAROM::BasisGenerator* bg, // Format string prec.str(std::string()); prec << "%." << ctr << "f"; - sprintf(buffer, prec.str().c_str(), energy_fractions[i]); + sprintf(buffer, prec.str().c_str(), energy_fractions[i]); ctr++; // Output string @@ -1433,6 +1435,7 @@ int main(int argc, char *argv[]) delete ode_solver; delete pmesh; delete romop; + delete soper; totalTimer.Stop(); if (myid == 0) @@ -1740,6 +1743,8 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, V_xTv_0 = new CAROM::Vector(fdim / 2, false); V_x.transposeMult(V_v, V_xTV_v); V_x.transposeMult(*v0_fom_librom, V_xTv_0); + Vx_librom_temp = new CAROM::Vector(fdim / 2, true); + Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fdim / 2); } if (!hyperreduce) @@ -1807,13 +1812,14 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const { HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, model, - x0, V_x, V_v, x_librom, + x0, V_x, V_v, x_librom, Vx_librom_temp, Vx_temp, eqp_coef, eqp_DS_coef, rank, resEQP); } else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, model, x0, - V_x, V_v, x_librom, rank, resEQP); + V_x, V_v, x_librom, Vx_librom_temp, Vx_temp, + rank, resEQP); Vector recv(resEQP); MPI_Allreduce(resEQP.GetData(), recv.GetData(), resEQP.Size(), MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); @@ -2058,7 +2064,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, for (int jj = 0; jj < dim; ++jj) { index = jj + ii * dim; - DS_coef[index + (i * dof*dim) + (j * rw.size() * dof*dim)] = DS.Elem(ii,jj); + DS_coef[index + (i * dof * dim) + (j * rw.size() * dof * dim)] = DS.Elem(ii, jj); } } } @@ -2067,8 +2073,10 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, - CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res) + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, + CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, + const int rank, Vector &res) + { const int rxdim = V_x.numColumns(); const int rvdim = V_v.numColumns(); @@ -2098,8 +2106,6 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, const Operator *P = fesR->GetProlongationMatrix(); // Vectors to be prolongated - CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); - Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); Vector Vx(fomdim); Vector vj(fomdim); @@ -2201,10 +2207,9 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } } - void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, - const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, + const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, Vector const &coef, Vector const &DS_coef, const int rank, Vector &res) { @@ -2235,8 +2240,6 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes const Operator *P = fesR->GetProlongationMatrix(); // Vectors to be prolongated - CAROM::Vector *Vx_librom_temp = new CAROM::Vector(fomdim, true); - Vector *Vx_temp = new Vector(&((*Vx_librom_temp)(0)), fomdim); Vector Vx(fomdim); // Prolongated vectors @@ -2311,7 +2314,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes for (int jj = 0; jj < dim; ++jj) { index = jj + ii * dim; - DS.Elem(ii,jj) = DS_coef[index + (i * elvect.Size()) + (j * qp.size() * elvect.Size())]; + DS.Elem(ii, jj) = DS_coef[index + (i * elvect.Size()) + (j * qp.size() * elvect.Size())]; } } @@ -2383,7 +2386,6 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, } } - void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, CAROM::Vector const &w, CAROM::Matrix &Gt, CAROM::Vector &sol) @@ -2437,7 +2439,6 @@ void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, cout << rank << ": relative residual norm for NNLS solution of Gs = Gw: " << relNorm << endl; } - // Compute EQP solution from constraints on snapshots. void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, ParFiniteElementSpace *fespace_X, @@ -2595,7 +2596,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } } -// Utility functions +// Utility functions void WriteMeshEQP(ParMesh *pmesh, const int myid, const int nqe, CAROM::Vector const &eqpSol) { @@ -2654,7 +2655,6 @@ void get_window_ids(int n_step, int n_window, CAROM::Vector *ids) } } - bool fileExists(const std::string &filename) { std::ifstream file(filename); diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index da7b6b7f3..4a8432f34 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -62,13 +62,14 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, // Perform hyperreduction with EQP void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, - CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res); + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, + CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, + const int rank, Vector &res); // Optimized EQP hyperreduction routine with preallocated arrays void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, - const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, + const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, Vector const &coef, Vector const &DS_coef, const int rank, Vector &res); // Compute a row in the G matrix which corresponds to a given FE element From a80d8684f5d8b32f321d91e75bdef16b3afc364c Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 29 Aug 2023 18:03:52 -0400 Subject: [PATCH 078/120] Cause of segfault error found --- examples/prom/nonlinear_elasticity_global_rom.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 245e6ff4f..5f1938ad2 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2073,7 +2073,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, const int rank, Vector &res) @@ -2145,9 +2145,9 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, for (int k = 0; k < V_v.numRows(); ++k) vj[k] = V_v(k, j); P->Mult(vj, p_vj); - res[j] = 0.0; eprev = -1; + double temp = 0.0; // For every quadrature weight for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 @@ -2201,9 +2201,10 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, // Calculate r[i] = ve_j^T * elvect for (int k = 0; k < elvect.Size(); k++) { - res[j] += vj_e[k] * elvect[k]; + temp += vj_e[k] * elvect[k]; } } + res[j] = temp; // This line causes segfault after error calculation } } @@ -2271,8 +2272,8 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes for (int j = 0; j < rvdim; ++j) { - eprev = -1; + double temp = 0.0; // For every quadrature weight for (int i = 0; i < qp.size(); ++i) // NOTE: i < 9 @@ -2324,11 +2325,13 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes // Calculate r[i] = ve_j^T * elvect // coef is size len(vdofs) * rvdim * rw.size + for (int k = 0; k < elvect.Size(); k++) { - res[j] += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * elvect[k]; + temp += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * elvect[k]; } } + res[j] = temp; // This line causes segfault after error calculation } } From fbca02291d278d35954ced2ba06395cf05cd6ec9 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 30 Aug 2023 17:39:18 -0400 Subject: [PATCH 079/120] Segfault fixed --- .../prom/nonlinear_elasticity_global_rom.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 5f1938ad2..2d3bb0da1 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1804,7 +1804,6 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const if (eqp) { // Lift v-vector and save - V_xTV_v->mult(v_librom, *pfom_v_librom); pfom_v_librom->plus(*V_xTv_0, dx_dt_librom); Vector resEQP; @@ -1832,8 +1831,8 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const // approximated by the reduced quadrature rule in the FOM space. // Therefore, the residual here is of dimension rxdim. z = 0.0; - MFEM_VERIFY(resEQP.Size() == rxdim, ""); - for (int i = 0; i < rxdim; ++i) + MFEM_VERIFY(resEQP.Size() == rvdim, ""); + for (int i = 0; i < rvdim; ++i) z[i] += resEQP[i]; } else @@ -2095,7 +2094,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, const FiniteElement *fe = NULL; Array vdofs; - res.SetSize(rxdim); + res.SetSize(rvdim); res = 0.0; int eprev = -1; @@ -2204,7 +2203,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, temp += vj_e[k] * elvect[k]; } } - res[j] = temp; // This line causes segfault after error calculation + res[j] = temp; } } @@ -2230,7 +2229,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes const FiniteElement *fe = NULL; Array vdofs; - res.SetSize(rxdim); + res.SetSize(rvdim); res = 0.0; int eprev = -1; @@ -2269,8 +2268,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes PMatO.UseExternalData(elvect.GetData(), dof, dim); // For every basis vector - for (int j = 0; j < rvdim; ++j) - + for (int j = 0; j < rvdim; ++j) { eprev = -1; double temp = 0.0; @@ -2331,7 +2329,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes temp += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * elvect[k]; } } - res[j] = temp; // This line causes segfault after error calculation + res[j] = temp; } } From 046b3a09d952a2eeb638ceacb15c7b7dbb9354fe Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 1 Sep 2023 13:43:54 -0400 Subject: [PATCH 080/120] Removed EQP bug --- examples/prom/nonlinear_elasticity_global_rom.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 2d3bb0da1..c8e6fde6e 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2467,6 +2467,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, CAROM::Matrix Gt(NQ, NB * nsnap, true); int current_size = 0; int i_start = 0; + int i_end = 0; int outer_loop_length = 0; if (!window_ids) { @@ -2504,20 +2505,22 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, MFEM_ABORT("P is null, generalize to serial case") } + // Outer loop for time windowing for (int oi = 0; oi < outer_loop_length; ++oi) { if (window_ids) { - i_start = window_ids->item(oi); - current_size = window_ids->item(oi + 1) - window_ids->item(oi) + 1; + i_start = window_ids->item(oi) - 1; + i_end = window_ids->item(oi + 1) - 1; + current_size = i_end - i_start; cout << "i_start = " << i_start << endl; cout << "Number of NNLS equations: " << NB * current_size << endl; Gt.setSize(NQ, NB * current_size); } // For every snapshot in batch - for (int i = i_start; i < current_size; ++i) + for (int i = i_start; i < i_end; ++i) { // Set the sampled dofs from the snapshot matrix for (int j = 0; j < BX_snapshots->numRows(); ++j) @@ -2654,6 +2657,7 @@ void get_window_ids(int n_step, int n_window, CAROM::Vector *ids) } ids->item(i - 1) += ctr; } + ids->item(n_window ) = n_step + 1; } bool fileExists(const std::string &filename) From 566316463d1e585f3a0367f603aca0b915dc9b63 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 1 Sep 2023 14:38:57 -0400 Subject: [PATCH 081/120] Add x0 to setupEQP --- examples/prom/nonlinear_elasticity_global_rom.cpp | 9 ++++----- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index c8e6fde6e..29b3af8ea 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -945,7 +945,7 @@ int main(int argc, char *argv[]) // EQP setup eqpSol = new CAROM::Vector(ir0->GetNPoints() * fespace.GetNE(), true); SetupEQP_snapshots(ir0, myid, &fespace, nsets, BV_librom, - GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "X"), + GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "X"), vx0.GetBlock(0), preconditionNNLS, tolNNLS, maxNNLSnnz, model, *eqpSol, window_ids); if (writeSampleMesh) @@ -1204,7 +1204,7 @@ int main(int argc, char *argv[]) if (use_eqp && window_ids && current_window < n_windows && ti == window_ids->item(current_window)) { // Load eqp and reinitialize ROM operator - cout << "Time window start at" << ti << endl; + cout << "Time window start at " << ti << endl; get_EQPsol(current_window, load_eqpsol); romop->SetEQP(load_eqpsol); ode_solver->Init(*romop); @@ -2444,7 +2444,7 @@ void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, ParFiniteElementSpace *fespace_X, const int nsets, const CAROM::Matrix *BV, - const CAROM::Matrix *BX_snapshots, + const CAROM::Matrix *BX_snapshots, const Vector x0, const bool precondition, const double nnls_tol, const int maxNNLSnnz, NeoHookeanModel *model, CAROM::Vector &sol, CAROM::Vector *window_ids) @@ -2505,7 +2505,6 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, MFEM_ABORT("P is null, generalize to serial case") } - // Outer loop for time windowing for (int oi = 0; oi < outer_loop_length; ++oi) { @@ -2657,7 +2656,7 @@ void get_window_ids(int n_step, int n_window, CAROM::Vector *ids) } ids->item(i - 1) += ctr; } - ids->item(n_window ) = n_step + 1; + ids->item(n_window) = n_step + 1; } bool fileExists(const std::string &filename) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 4a8432f34..6e05740b9 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -85,7 +85,7 @@ void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, ParFiniteElementSpace *fespace_X, const int nsets, const CAROM::Matrix *BV, - const CAROM::Matrix *BX_snapshots, + const CAROM::Matrix *BX_snapshots, const Vector x0, const bool precondition, const double nnls_tol, const int maxNNLSnnz, NeoHookeanModel *model, CAROM::Vector &sol, CAROM::Vector *window_ids); From 9be15be1972aa50759b1aa1e3a387cada934514b Mon Sep 17 00:00:00 2001 From: Chris Vales Date: Fri, 1 Sep 2023 14:48:04 -0400 Subject: [PATCH 082/120] add matrix row/col rescaling methods --- lib/linalg/Matrix.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/linalg/Matrix.h b/lib/linalg/Matrix.h index 4f7f74aba..233b6cc67 100644 --- a/lib/linalg/Matrix.h +++ b/lib/linalg/Matrix.h @@ -946,6 +946,18 @@ class Matrix void orthogonalize(); + /** + * @brief Rescale every matrix row by its maximum absolute value. + */ + void + rescale_rows_max(); + + /** + * @brief Rescale every matrix column by its maximum absolute value. + */ + void + rescale_cols_max(); + /** * @brief Rescale every matrix row by its maximum absolute value. */ From 23107d6d8264567349a220e4d26e3e828f98c25a Mon Sep 17 00:00:00 2001 From: Chris Vales Date: Fri, 1 Sep 2023 14:48:52 -0400 Subject: [PATCH 083/120] normalize NNLS constraints --- examples/prom/nonlinear_elasticity_global_rom.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 29b3af8ea..ee387a656 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2580,6 +2580,9 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } } // Loop (i) over snapshots + // Rescale every Gt column (NNLS equation) by its max absolute value. + Gt.rescale_cols_max(); + Array const &w_el = ir0->GetWeights(); MFEM_VERIFY(w_el.Size() == nqe, ""); From 11b0d6d21588e99c467df0154c819e97dc7e29ce Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 1 Sep 2023 15:02:26 -0400 Subject: [PATCH 084/120] Added x0 --- examples/prom/nonlinear_elasticity_global_rom.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index ee387a656..e660bb491 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2523,7 +2523,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, { // Set the sampled dofs from the snapshot matrix for (int j = 0; j < BX_snapshots->numRows(); ++j) - x_i[j] = (*BX_snapshots)(j, i); + x_i[j] = (*BX_snapshots)(j, i) + x0[j]; // Get prolongated dofs Vector px_i; From 7e6191eeb887a6590086d1fa0bc3306b1520455f Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 19 Sep 2023 18:25:29 -0400 Subject: [PATCH 085/120] Fixed bug in NNLS constraint forming loop. --- examples/prom/nonlinear_elasticity_global_rom.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index e660bb491..dd131d1d6 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2473,6 +2473,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, { current_size = nsnap; outer_loop_length = 1; + i_end = nsnap; } else { From eb1e865bc5077137698edda96cf206535930cd0b Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 20 Sep 2023 17:38:55 -0400 Subject: [PATCH 086/120] Max iters increased --- examples/prom/nonlinear_elasticity_global_rom.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index dd131d1d6..9572c8d00 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1505,7 +1505,7 @@ HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f, M_solver.iterative_mode = false; M_solver.SetRelTol(rel_tol); M_solver.SetAbsTol(0.0); - M_solver.SetMaxIter(30); + M_solver.SetMaxIter(1000); M_solver.SetPrintLevel(0); M_prec.SetType(HypreSmoother::Jacobi); M_solver.SetPreconditioner(M_prec); @@ -1669,6 +1669,9 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, S_hat_v0_temp->Size(), true, false); M_hat = new CAROM::Matrix(rvdim, rvdim, false); M_hat_inv = new CAROM::Matrix(rvdim, rvdim, false); + + // Set the max iterations for the mass matrix solver + M_hat_solver.SetMaxIter(1000); // Create S_hat ComputeCtAB(fom->Smat, V_v, V_v, *S_hat); From d0fc14e3ddd2737b278ec0d0884d1a8f07ebe833 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 4 Oct 2023 15:03:18 -0400 Subject: [PATCH 087/120] improved eqp speed --- .../prom/nonlinear_elasticity_global_rom.cpp | 326 +++++++++--------- 1 file changed, 161 insertions(+), 165 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 9572c8d00..2c5b212e6 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1669,7 +1669,7 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, S_hat_v0_temp->Size(), true, false); M_hat = new CAROM::Matrix(rvdim, rvdim, false); M_hat_inv = new CAROM::Matrix(rvdim, rvdim, false); - + // Set the max iterations for the mass matrix solver M_hat_solver.SetMaxIter(1000); @@ -2009,66 +2009,64 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, DenseMatrix DSh(dof, dim); DenseMatrix DS(dof, dim); DenseMatrix Jrt(dim); - DS_coef.SetSize(dof * dim * rw.size() * rvdim); + DS_coef.SetSize(dof * dim * rw.size()); DS_coef = 0.0; int index = 0; - - // For every basis vector - for (int j = 0; j < rvdim; ++j) + eprev = -1; + // For every quadrature weight + for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 { - // Get basis vector and prolongate - for (int k = 0; k < V_v.numRows(); ++k) - vj[k] = V_v(k, j); - P->Mult(vj, p_vj); - - eprev = -1; + const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); - // For every quadrature weight - for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 + if (e != eprev) // Update element transformation { - const int e = qp[i] / nqe; // Element index - // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); - const IntegrationPoint &ip = ir->IntPoint(qpi); + doftrans = fesR->GetElementVDofs(e, vdofs); + eltrans = fesR->GetElementTransformation(e); - if (e != eprev) // Update element transformation + if (doftrans) { - doftrans = fesR->GetElementVDofs(e, vdofs); - eltrans = fesR->GetElementTransformation(e); + MFEM_ABORT("TODO"); + } + eprev = e; + } - if (doftrans) - { - MFEM_ABORT("TODO"); - } + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); + // Get the transformation weight + double t = eltrans->Weight(); - // Get element vectors - p_vj.GetSubVector(vdofs, vj_e); - eprev = e; + // Calculate DS and store + CalcInverse(eltrans->Jacobian(), Jrt); + fe->CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + for (int ii = 0; ii < dof; ++ii) + { + for (int jj = 0; jj < dim; ++jj) + { + index = jj + ii * dim; + DS_coef[index + (i * dof * dim)] = DS.Elem(ii, jj); } + } + // For every basis vector + for (int j = 0; j < rvdim; ++j) + { + // Get basis vector and prolongate + for (int k = 0; k < V_v.numRows(); ++k) + vj[k] = V_v(k, j); + P->Mult(vj, p_vj); + + // Get element vectors + p_vj.GetSubVector(vdofs, vj_e); - // Set integration point in the element transformation - eltrans->SetIntPoint(&ip); - model->SetTransformation(*eltrans); - // Get the transformation weight - double t = eltrans->Weight(); // Calculate r[i] = ve_j^T * elvect for (int k = 0; k < elvect_size; k++) { coef[k + (i * elvect_size) + (j * rw.size() * elvect_size)] = vj_e[k] * rw[i] * t; } - - // Calculate DS and store - CalcInverse(eltrans->Jacobian(), Jrt); - fe->CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - for (int ii = 0; ii < dof; ++ii) - { - for (int jj = 0; jj < dim; ++jj) - { - index = jj + ii * dim; - DS_coef[index + (i * dof * dim) + (j * rw.size() * dof * dim)] = DS.Elem(ii, jj); - } - } } } } @@ -2139,74 +2137,74 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, Vector elvect(dof * dim); PMatO.UseExternalData(elvect.GetData(), dof, dim); - // For every basis vector - for (int j = 0; j < rvdim; ++j) - + eprev = -1; + // For every quadrature weight + for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 { - // Get basis vector and prolongate - for (int k = 0; k < V_v.numRows(); ++k) - vj[k] = V_v(k, j); - P->Mult(vj, p_vj); + const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); - eprev = -1; - double temp = 0.0; - - // For every quadrature weight - for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 + if (e != eprev) // Update element transformation { - const int e = qp[i] / nqe; // Element index - // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); - const IntegrationPoint &ip = ir->IntPoint(qpi); + doftrans = fesR->GetElementVDofs(e, vdofs); + fe = fesR->GetFE(e); + eltrans = fesR->GetElementTransformation(e); + + dof = fe->GetDof(); // Get number of dofs in element + dim = fe->GetDim(); - if (e != eprev) // Update element transformation + if (doftrans) { - doftrans = fesR->GetElementVDofs(e, vdofs); - fe = fesR->GetFE(e); - eltrans = fesR->GetElementTransformation(e); + MFEM_ABORT("TODO"); + } - dof = fe->GetDof(); // Get number of dofs in element - dim = fe->GetDim(); + // Get element vectors + p_Vx.GetSubVector(vdofs, Vx_e); + eprev = e; + } - if (doftrans) - { - MFEM_ABORT("TODO"); - } + // Integration at ip + PMatI.UseExternalData(Vx_e.GetData(), dof, dim); - // Get element vectors - p_Vx.GetSubVector(vdofs, Vx_e); - p_vj.GetSubVector(vdofs, vj_e); - eprev = e; - } + elvect = 0.0; + + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); - // Integration at ip - PMatI.UseExternalData(Vx_e.GetData(), dof, dim); + // Get the transformation weight + double t = eltrans->Weight(); - elvect = 0.0; + // Compute action of nonlinear operator + CalcInverse(eltrans->Jacobian(), Jrt); + fe->CalcDShape(ip, DSh); + Mult(DSh, Jrt, DS); + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P_f); + P_f *= (t * rw[i]); // NB: Not by ip.weight + AddMultABt(DS, P_f, PMatO); - // Set integration point in the element transformation - eltrans->SetIntPoint(&ip); - model->SetTransformation(*eltrans); + // For every basis vector + for (int j = 0; j < rvdim; ++j) + { + // Get basis vector and prolongate + for (int k = 0; k < V_v.numRows(); ++k) + vj[k] = V_v(k, j); + P->Mult(vj, p_vj); - // Get the transformation weight - double t = eltrans->Weight(); + p_vj.GetSubVector(vdofs, vj_e); - // Compute action of nonlinear operator - CalcInverse(eltrans->Jacobian(), Jrt); - fe->CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P_f); - P_f *= (t * rw[i]); // NB: Not by ip.weight - AddMultABt(DS, P_f, PMatO); + double temp = 0.0; // Calculate r[i] = ve_j^T * elvect for (int k = 0; k < elvect.Size(); k++) { temp += vj_e[k] * elvect[k]; } + res[j] += temp; } - res[j] = temp; } } @@ -2270,69 +2268,68 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes Vector elvect(dof * dim); PMatO.UseExternalData(elvect.GetData(), dof, dim); - // For every basis vector - for (int j = 0; j < rvdim; ++j) + eprev = -1; + double temp = 0.0; + + // For every quadrature weight + for (int i = 0; i < qp.size(); ++i) // NOTE: i < 9 { - eprev = -1; - double temp = 0.0; + const int e = qp[i] / nqe; // Element index + // Local (element) index of the quadrature point + const int qpi = qp[i] - (e * nqe); + const IntegrationPoint &ip = ir->IntPoint(qpi); - // For every quadrature weight - for (int i = 0; i < qp.size(); ++i) // NOTE: i < 9 + if (e != eprev) // Update element transformation { - const int e = qp[i] / nqe; // Element index - // Local (element) index of the quadrature point - const int qpi = qp[i] - (e * nqe); - const IntegrationPoint &ip = ir->IntPoint(qpi); - - if (e != eprev) // Update element transformation - { - doftrans = fesR->GetElementVDofs(e, vdofs); - fe = fesR->GetFE(e); - eltrans = fesR->GetElementTransformation(e); - - dof = fe->GetDof(); // Get number of dofs in element - dim = fe->GetDim(); + doftrans = fesR->GetElementVDofs(e, vdofs); + fe = fesR->GetFE(e); + eltrans = fesR->GetElementTransformation(e); - if (doftrans) - { - MFEM_ABORT("TODO"); - } + dof = fe->GetDof(); // Get number of dofs in element + dim = fe->GetDim(); - // Get element vectors - p_Vx.GetSubVector(vdofs, Vx_e); - eprev = e; + if (doftrans) + { + MFEM_ABORT("TODO"); } - // Integration at ip - elvect = 0.0; - PMatI.UseExternalData(Vx_e.GetData(), dof, dim); + // Get element vectors + p_Vx.GetSubVector(vdofs, Vx_e); + eprev = e; + } + + // Integration at ip + elvect = 0.0; + PMatI.UseExternalData(Vx_e.GetData(), dof, dim); - // Set integration point in the element transformation - eltrans->SetIntPoint(&ip); - model->SetTransformation(*eltrans); + // Set integration point in the element transformation + eltrans->SetIntPoint(&ip); + model->SetTransformation(*eltrans); - for (int ii = 0; ii < dof; ++ii) + for (int ii = 0; ii < dof; ++ii) + { + for (int jj = 0; jj < dim; ++jj) { - for (int jj = 0; jj < dim; ++jj) - { - index = jj + ii * dim; - DS.Elem(ii, jj) = DS_coef[index + (i * elvect.Size()) + (j * qp.size() * elvect.Size())]; - } + index = jj + ii * dim; + DS.Elem(ii, jj) = DS_coef[index + (i * elvect.Size())]; } + } - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P_f); - AddMultABt(DS, P_f, PMatO); - - // Calculate r[i] = ve_j^T * elvect - // coef is size len(vdofs) * rvdim * rw.size + MultAtB(PMatI, DS, Jpt); + model->EvalP(Jpt, P_f); + AddMultABt(DS, P_f, PMatO); + // Calculate r[i] = ve_j^T * elvect + // coef is size len(vdofs) * rvdim * rw.size + // For every basis vector TODO: This should not be dependent on j + for (int j = 0; j < rvdim; ++j) + { for (int k = 0; k < elvect.Size(); k++) { temp += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * elvect[k]; } + res[j] += temp; } - res[j] = temp; } } @@ -2539,37 +2536,36 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, if (skipFirstW && i > 0 && i % nsnapPerSet == 0) skip++; - // For each basis vector - for (int j = 0; j < NB; ++j) + // TODO: is it better to make the element loop the outer loop? + // For each element + for (int e = 0; e < ne; ++e) { + // Get element and its dofs and transformation. + Array vdofs; + DofTransformation *doftrans = fespace_X->GetElementVDofs(e, vdofs); + const FiniteElement &fe = *fespace_X->GetFE(e); + ElementTransformation *eltrans = fespace_X->GetElementTransformation(e); + px_i.GetSubVector(vdofs, elfun); - // Get basis vector - for (int k = 0; k < BV->numRows(); ++k) - v_j[k] = (*BV)(k, j); + if (doftrans) + { + MFEM_ABORT("Doftrans is true, make corresponding edits") + } - // Get prolongated dofs - Vector pv_j; - Vector ve_j; + // For each basis vector + for (int j = 0; j < NB; ++j) + { + // Get basis vector + for (int k = 0; k < BV->numRows(); ++k) + v_j[k] = (*BV)(k, j); - pv_j.SetSize(P->Height()); - P->Mult(v_j, pv_j); + // Get prolongated dofs + Vector pv_j; + Vector ve_j; - // TODO: is it better to make the element loop the outer loop? - // For each element - for (int e = 0; e < ne; ++e) - { - // Get element and its dofs and transformation. - Array vdofs; - DofTransformation *doftrans = fespace_X->GetElementVDofs(e, vdofs); - const FiniteElement &fe = *fespace_X->GetFE(e); - ElementTransformation *eltrans = fespace_X->GetElementTransformation(e); - px_i.GetSubVector(vdofs, elfun); + pv_j.SetSize(P->Height()); + P->Mult(v_j, pv_j); pv_j.GetSubVector(vdofs, ve_j); - if (doftrans) - { - MFEM_ABORT("Doftrans is true, make corresponding edits") - } - // Compute the row of G corresponding to element e, store in r ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r); @@ -2584,8 +2580,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } } // Loop (i) over snapshots - // Rescale every Gt column (NNLS equation) by its max absolute value. - Gt.rescale_cols_max(); + // Rescale every Gt column (NNLS equation) by its max absolute value. + Gt.rescale_cols_max(); Array const &w_el = ir0->GetWeights(); MFEM_VERIFY(w_el.Size() == nqe, ""); From ca4e7fcf5d01e29bd7f3f96cae17fe32363bad84 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 5 Oct 2023 15:55:41 -0400 Subject: [PATCH 088/120] resetting temp variable now --- examples/prom/nonlinear_elasticity_global_rom.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 2c5b212e6..20a21ddf2 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2138,6 +2138,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, PMatO.UseExternalData(elvect.GetData(), dof, dim); eprev = -1; + double temp = 0.0; // For every quadrature weight for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 { @@ -2185,7 +2186,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, model->EvalP(Jpt, P_f); P_f *= (t * rw[i]); // NB: Not by ip.weight AddMultABt(DS, P_f, PMatO); - + // For every basis vector for (int j = 0; j < rvdim; ++j) { @@ -2196,7 +2197,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, p_vj.GetSubVector(vdofs, vj_e); - double temp = 0.0; + temp = 0.0; // Calculate r[i] = ve_j^T * elvect for (int k = 0; k < elvect.Size(); k++) @@ -2322,6 +2323,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes // Calculate r[i] = ve_j^T * elvect // coef is size len(vdofs) * rvdim * rw.size // For every basis vector TODO: This should not be dependent on j + temp = 0.0; for (int j = 0; j < rvdim; ++j) { for (int k = 0; k < elvect.Size(); k++) From 0fe6c74b4ca7ccc0f2546fd803998097178fbdca Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 5 Oct 2023 16:32:33 -0400 Subject: [PATCH 089/120] Moved temporary variable to the correct position --- examples/prom/nonlinear_elasticity_global_rom.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 20a21ddf2..c32037e36 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2186,7 +2186,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, model->EvalP(Jpt, P_f); P_f *= (t * rw[i]); // NB: Not by ip.weight AddMultABt(DS, P_f, PMatO); - + // For every basis vector for (int j = 0; j < rvdim; ++j) { @@ -2323,9 +2323,9 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes // Calculate r[i] = ve_j^T * elvect // coef is size len(vdofs) * rvdim * rw.size // For every basis vector TODO: This should not be dependent on j - temp = 0.0; for (int j = 0; j < rvdim; ++j) { + temp = 0.0; for (int k = 0; k < elvect.Size(); k++) { temp += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * elvect[k]; From 5f39f07d62b26f7f8ed28be3edd08b8f530cadae Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 23 Oct 2023 22:02:15 -0400 Subject: [PATCH 090/120] Minor formatting changes --- .../prom/nonlinear_elasticity_global_rom.cpp | 17 ++++++----------- .../nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index c32037e36..190260897 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -920,7 +920,6 @@ int main(int argc, char *argv[]) printf("reduced H dim = %d\n", hdim); // Setup hyperreduction, using either EQP or sampled DOFs and a sample mesh. - ParFiniteElementSpace *sp_XV_space; CAROM::Matrix *Hsinv = new CAROM::Matrix(hdim, hdim, false); // Gets resized before use. const IntegrationRule *ir0 = NULL; @@ -1121,7 +1120,6 @@ int main(int argc, char *argv[]) { if (!use_eqp) { - // Define operator in sample space soper = new HyperelasticOperator(*sp_XV_space, ess_tdof_list_sp, visc, mu, K); } @@ -1131,7 +1129,7 @@ int main(int argc, char *argv[]) } } - if (!use_eqp) // TODO: ask about whether this is needed. + if (!use_eqp) { romop = new RomOperator(&oper, soper, rvdim, rxdim, hdim, smm, w_v0, w_x0, vx0.GetBlock(0), BV_librom, BX_librom, H_librom, Hsinv, myid, @@ -1228,7 +1226,6 @@ int main(int argc, char *argv[]) if (offline) { - if (basis_generator_x->isNextSample(t) || x_base_only == false && basis_generator_v->isNextSample(t)) { dvxdt = oper.dvxdt_sp.GetData(); @@ -2028,7 +2025,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, if (doftrans) { - MFEM_ABORT("TODO"); + MFEM_ABORT("TODO: Implementation for when `doftrans` is not NULL"); } eprev = e; } @@ -2062,7 +2059,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, // Get element vectors p_vj.GetSubVector(vdofs, vj_e); - // Calculate r[i] = ve_j^T * elvect + // Calculate coeff for (int k = 0; k < elvect_size; k++) { coef[k + (i * elvect_size) + (j * rw.size() * elvect_size)] = vj_e[k] * rw[i] * t; @@ -2158,7 +2155,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, if (doftrans) { - MFEM_ABORT("TODO"); + MFEM_ABORT("TODO: Implementation for when `doftrans` is not NULL"); } // Get element vectors @@ -2214,7 +2211,6 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, Vector const &coef, Vector const &DS_coef, const int rank, Vector &res) { - const int rxdim = V_x.numColumns(); const int rvdim = V_v.numColumns(); const int fomdim = V_x.numRows(); @@ -2291,7 +2287,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes if (doftrans) { - MFEM_ABORT("TODO"); + MFEM_ABORT("TODO: Implementation for when `doftrans` is not NULL"); } // Get element vectors @@ -2322,7 +2318,6 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes // Calculate r[i] = ve_j^T * elvect // coef is size len(vdofs) * rvdim * rw.size - // For every basis vector TODO: This should not be dependent on j for (int j = 0; j < rvdim; ++j) { temp = 0.0; @@ -2578,7 +2573,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, if (precondition) { - MFEM_ABORT("TODO"); + MFEM_ABORT("TODO: Implement preconditioned NNLS for this example"); } } // Loop (i) over snapshots diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 6e05740b9..f44621773 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -62,7 +62,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, // Perform hyperreduction with EQP void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, const int rank, Vector &res); From 871179dbd129fecb9dd608ca2cd96130ca92a75f Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Mon, 23 Oct 2023 23:12:40 -0400 Subject: [PATCH 091/120] Removed duplicate lines in examples --- .../prom/nonlinear_elasticity_global_rom.cpp | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 190260897..300364572 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -31,7 +31,6 @@ // // Offline phase: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.9 -id 0 -// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.9 -id 0 // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.1 -id 1 // // Merge phase: @@ -39,7 +38,6 @@ // // Create FOM comparison data: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -id 2 -// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -id 2 // // Online phase with full sampling: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rvdim 40 -rxdim 10 -hdim 71 -nsr 1170 -sc 1.0 @@ -76,15 +74,12 @@ // Offline phase: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.9 -xbo -def-ic -id 0 // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.1 -xbo -def-ic -id 1 -// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 0.9 -xbo -def-ic -id 0 -// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.1 -xbo -def-ic -id 1 // // Merge phase: // ./nonlinear_elasticity_global_rom -merge -ns 2 -dt 0.01 -tf 5.0 -xbo // // Create FOM comparison data: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -xbo -def-ic -id 2 -// ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -xbo -def-ic -id 2 // // Online phase with full sampling: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rxdim 57 -hdim 183 -nsr 1170 -sc 1.0 -xbo -def-ic @@ -107,20 +102,6 @@ // Relative error of ROM position (x) at t_final: 5 is 0.0108709 // Relative error of ROM velocity (v) at t_final: 5 is 1.30704 // -// Online phase with strong hyper reduction, using QDEIM: -// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype qdeim -rxdim 2 -hdim 4 -nsr 10 -sc 1.0 -xbo -def-ic -// Output message: -// Elapsed time for time integration loop 0.10806 -// Relative error of ROM position (x) at t_final: 5 is 0.0108709 -// Relative error of ROM velocity (v) at t_final: 5 is 1.30704 -// -// Online phase with strong hyper reduction, using QDEIM: -// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype qdeim -rxdim 2 -hdim 4 -nsr 10 -sc 1.0 -xbo -def-ic -// Output message: -// Elapsed time for time integration loop 0.10806 -// Relative error of ROM position (x) at t_final: 5 is 0.0108709 -// Relative error of ROM velocity (v) at t_final: 5 is 1.30704 -// // Online phase with EQP hyper reduction: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -rxdim 2 -hdim 1 -ntw 25 -sc 1.00 -xbo -def-ic // Elapsed time for time integration loop 0.766614 From 33c79547e8392b78b4d3aa0a9b1ff7fc35f6e89f Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:06:03 -0400 Subject: [PATCH 092/120] build succeeds --- lib/linalg/Matrix.h | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/lib/linalg/Matrix.h b/lib/linalg/Matrix.h index 233b6cc67..bf68b33b8 100644 --- a/lib/linalg/Matrix.h +++ b/lib/linalg/Matrix.h @@ -958,18 +958,6 @@ class Matrix void rescale_cols_max(); - /** - * @brief Rescale every matrix row by its maximum absolute value. - */ - void - rescale_rows_max(); - - /** - * @brief Rescale every matrix column by its maximum absolute value. - */ - void - rescale_cols_max(); - /** * @brief Const Matrix member access. Matrix data is stored in * row-major format. From fcfd87acc28e85be4f5b0b4cc517551064d5ffc0 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:31:35 -0400 Subject: [PATCH 093/120] Updated formatting --- .../prom/nonlinear_elasticity_global_rom.cpp | 141 ++++++++++-------- .../nonlinear_elasticity_global_rom_eqp.hpp | 29 ++-- 2 files changed, 98 insertions(+), 72 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 300364572..64981f354 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -213,7 +213,8 @@ class RomOperator : public TimeDependentOperator const Vector *x0_, const Vector v0_fom_, const CAROM::Matrix *V_v_, const CAROM::Matrix *V_x_, const CAROM::Matrix *U_H_, const CAROM::Matrix *Hsinv_, const int myid, const bool oversampling_, - const bool hyperreduce_, const bool x_base_only_, const bool use_eqp, CAROM::Vector *eqpSol, + const bool hyperreduce_, const bool x_base_only_, const bool use_eqp, + CAROM::Vector *eqpSol, const IntegrationRule *ir_eqp_, NeoHookeanModel *model_); virtual void Mult(const Vector &y, Vector &dy_dt) const; @@ -330,7 +331,8 @@ void BasisGeneratorFinalSummary(CAROM::BasisGenerator* bg, if (!reached_cutoff) cutoff = sing_vals->dim(); - outfile << "Take first " << cutoff << " of " << sing_vals->dim() << " basis vectors" << endl; + outfile << "Take first " << cutoff << " of " << sing_vals->dim() << + " basis vectors" << endl; outfile.close(); } @@ -347,7 +349,8 @@ void MergeBasis(const int dimFOM, const int nparam, const int max_num_snapshots, for (int paramID = 0; paramID < nparam; ++paramID) { - std::string snapshot_filename = "basis" + std::to_string(paramID) + "_" + name + "_snapshot"; + std::string snapshot_filename = "basis" + std::to_string( + paramID) + "_" + name + "_snapshot"; generator.loadSamples(snapshot_filename, "snapshot"); } @@ -371,7 +374,8 @@ const CAROM::Matrix *GetSnapshotMatrix(const int dimFOM, const int nparam, for (int paramID = 0; paramID < nparam; ++paramID) { - std::string snapshot_filename = "basis" + std::to_string(paramID) + "_" + name + "_snapshot"; + std::string snapshot_filename = "basis" + std::to_string( + paramID) + "_" + name + "_snapshot"; generator.loadSamples(snapshot_filename, "snapshot"); } @@ -548,7 +552,8 @@ int main(int argc, char *argv[]) args.PrintOptions(cout); } - const bool check = (offline && !merge && !online) || (!offline && merge && !online) || (!offline && !merge && online); + const bool check = (offline && !merge && !online) || (!offline && merge + && !online) || (!offline && !merge && online); MFEM_VERIFY(check, "only one of offline, merge, or online must be true!"); StopWatch solveTimer, totalTimer; @@ -733,9 +738,9 @@ int main(int argc, char *argv[]) Vector *x_rec = new Vector(x_gf.GetTrueVector()); CAROM::Vector *v_rec_librom = new CAROM::Vector(v_rec->GetData(), v_rec->Size(), - true, false); + true, false); CAROM::Vector *x_rec_librom = new CAROM::Vector(x_rec->GetData(), x_rec->Size(), - true, false); + true, false); // 9. Initialize the hyperelastic operator, the GLVis visualization and print // the initial energies. @@ -811,14 +816,14 @@ int main(int argc, char *argv[]) if (x_base_only == false) { basis_generator_v = new CAROM::BasisGenerator(options, isIncremental, - basisFileName + "_V"); + basisFileName + "_V"); } basis_generator_x = new CAROM::BasisGenerator(options, isIncremental, - basisFileName + "_X"); + basisFileName + "_X"); basis_generator_H = new CAROM::BasisGenerator(options, isIncremental, - basisFileName + "_H"); + basisFileName + "_H"); } RomOperator *romop = 0; @@ -836,7 +841,8 @@ int main(int argc, char *argv[]) CAROM::Vector *eqpSol_S = nullptr; CAROM::Vector *window_ids = nullptr; - CAROM::Vector *load_eqpsol = new CAROM::Vector(1, false); // Will be resized later + CAROM::Vector *load_eqpsol = new CAROM::Vector(1, + false); // Will be resized later // 11. Initialize ROM operator // I guess most of this should be done on id =0 @@ -847,7 +853,8 @@ int main(int argc, char *argv[]) if (x_base_only) { - readerV = new CAROM::BasisReader("basisX"); // The basis for v uses the x basis instead. + readerV = new + CAROM::BasisReader("basisX"); // The basis for v uses the x basis instead. rvdim = rxdim; } else @@ -902,7 +909,8 @@ int main(int argc, char *argv[]) // Setup hyperreduction, using either EQP or sampled DOFs and a sample mesh. ParFiniteElementSpace *sp_XV_space; - CAROM::Matrix *Hsinv = new CAROM::Matrix(hdim, hdim, false); // Gets resized before use. + CAROM::Matrix *Hsinv = new CAROM::Matrix(hdim, hdim, + false); // Gets resized before use. const IntegrationRule *ir0 = NULL; if (ir0 == NULL) @@ -925,7 +933,8 @@ int main(int argc, char *argv[]) // EQP setup eqpSol = new CAROM::Vector(ir0->GetNPoints() * fespace.GetNE(), true); SetupEQP_snapshots(ir0, myid, &fespace, nsets, BV_librom, - GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "X"), vx0.GetBlock(0), + GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "X"), + vx0.GetBlock(0), preconditionNNLS, tolNNLS, maxNNLSnnz, model, *eqpSol, window_ids); if (writeSampleMesh) @@ -944,19 +953,19 @@ int main(int argc, char *argv[]) nsamp_H = hdim; } - CAROM::Matrix* Hsinv = new CAROM::Matrix(nsamp_H, hdim, false); - vector sample_dofs(nsamp_H); + CAROM::Matrix* Hsinv = new CAROM::Matrix(nsamp_H, hdim, false); + vector sample_dofs(nsamp_H); - // Setup hyperreduction using DEIM, GNAT, or S-OPT - CAROM::Hyperreduction hr(samplingType); - hr.ComputeSamples(H_librom, - hdim, - sample_dofs, - num_sample_dofs_per_proc, - *Hsinv, - myid, - num_procs, - nsamp_H); + // Setup hyperreduction using DEIM, GNAT, or S-OPT + CAROM::Hyperreduction hr(samplingType); + hr.ComputeSamples(H_librom, + hdim, + sample_dofs, + num_sample_dofs_per_proc, + *Hsinv, + myid, + num_procs, + nsamp_H); // Construct sample mesh const int nspaces = 1; @@ -1180,7 +1189,8 @@ int main(int argc, char *argv[]) { if (myid == 0) { - if (use_eqp && window_ids && current_window < n_windows && ti == window_ids->item(current_window)) + if (use_eqp && window_ids && current_window < n_windows + && ti == window_ids->item(current_window)) { // Load eqp and reinitialize ROM operator cout << "Time window start at " << ti << endl; @@ -1207,7 +1217,8 @@ int main(int argc, char *argv[]) if (offline) { - if (basis_generator_x->isNextSample(t) || x_base_only == false && basis_generator_v->isNextSample(t)) + if (basis_generator_x->isNextSample(t) || x_base_only == false + && basis_generator_v->isNextSample(t)) { dvxdt = oper.dvxdt_sp.GetData(); vx_diff = BlockVector(vx); @@ -1297,7 +1308,8 @@ int main(int argc, char *argv[]) } // timestep loop if (myid == 0) - cout << "Elapsed time for time integration loop " << solveTimer.RealTime() << endl; + cout << "Elapsed time for time integration loop " << solveTimer.RealTime() << + endl; ostringstream velo_name, pos_name; @@ -1396,9 +1408,9 @@ int main(int argc, char *argv[]) double tot_diff_norm_x = sqrt(InnerProduct(MPI_COMM_WORLD, diff_x, diff_x)); double tot_v_fom_norm = sqrt(InnerProduct(MPI_COMM_WORLD, - v_fom, v_fom)); + v_fom, v_fom)); double tot_x_fom_norm = sqrt(InnerProduct(MPI_COMM_WORLD, - x_fom, x_fom)); + x_fom, x_fom)); if (myid == 0) { @@ -1458,8 +1470,8 @@ void visualize(ostream &out, ParMesh *mesh, ParGridFunction *deformed_nodes, } HyperelasticOperator::HyperelasticOperator(ParFiniteElementSpace &f, - Array &ess_tdof_list_, double visc, - double mu, double K) + Array &ess_tdof_list_, double visc, + double mu, double K) : TimeDependentOperator(2 * f.TrueVSize(), 0.0), fespace(f), ess_tdof_list(ess_tdof_list_), M(NULL), S(NULL), H(NULL), @@ -1607,7 +1619,8 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, rxdim(rxdim_), rvdim(rvdim_), hdim(hdim_), x0(x0_), v0(v0_), v0_fom(v0_fom_), smm(smm_), V_x(*V_x_), V_v(*V_v_), U_H(U_H_), Hsinv(Hsinv_), M_hat_solver(fom_->fespace.GetComm()), oversampling(oversampling_), - z(height / 2), hyperreduce(hyperreduce_), x_base_only(x_base_only_), eqp(use_eqp), + z(height / 2), hyperreduce(hyperreduce_), x_base_only(x_base_only_), + eqp(use_eqp), ir_eqp(ir_eqp_), model(model_), rank(myid) { if (!eqp) @@ -1644,7 +1657,7 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, S_hat_v0 = new CAROM::Vector(rvdim, false); S_hat_v0_temp = new Vector(v0_fom.Size()); S_hat_v0_temp_librom = new CAROM::Vector(S_hat_v0_temp->GetData(), - S_hat_v0_temp->Size(), true, false); + S_hat_v0_temp->Size(), true, false); M_hat = new CAROM::Matrix(rvdim, rvdim, false); M_hat_inv = new CAROM::Matrix(rvdim, rvdim, false); @@ -1758,7 +1771,7 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, << fom->fespace.GetNE() << endl; GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, - ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef); + ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef); } } @@ -1784,22 +1797,22 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const CAROM::Vector dx_dt_librom(dx_dt.GetData(), rxdim, false, false); if (eqp) - { // Lift v-vector and save + { // Lift v-vector and save V_xTV_v->mult(v_librom, *pfom_v_librom); pfom_v_librom->plus(*V_xTv_0, dx_dt_librom); Vector resEQP; if (fastIntegration) { HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), eqp_rw, - eqp_qp, ir_eqp, model, - x0, V_x, V_v, x_librom, Vx_librom_temp, Vx_temp, - eqp_coef, eqp_DS_coef, rank, resEQP); + eqp_qp, ir_eqp, model, + x0, V_x, V_v, x_librom, Vx_librom_temp, Vx_temp, + eqp_coef, eqp_DS_coef, rank, resEQP); } else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, - eqp_qp, ir_eqp, model, x0, - V_x, V_v, x_librom, Vx_librom_temp, Vx_temp, - rank, resEQP); + eqp_qp, ir_eqp, model, x0, + V_x, V_v, x_librom, Vx_librom_temp, Vx_temp, + rank, resEQP); Vector recv(resEQP); MPI_Allreduce(resEQP.GetData(), recv.GetData(), resEQP.Size(), MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); @@ -1931,7 +1944,7 @@ void RomOperator::SetEQP(CAROM::Vector *eqpSol) << fom->fespace.GetNE() << endl; GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, - ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef); + ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef); } // Functions for EQP functionality @@ -1939,9 +1952,9 @@ void RomOperator::SetEQP(CAROM::Vector *eqpSol) // Compute coefficients of the reduced integrator with respect to inputs Q and x // in HyperelasticNLFIntegrator_ComputeReducedEQP. void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, - std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, - CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef) + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, + CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef) { const int rvdim = V_v.numColumns(); const int fomdim = V_v.numRows(); @@ -2043,17 +2056,19 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, // Calculate coeff for (int k = 0; k < elvect_size; k++) { - coef[k + (i * elvect_size) + (j * rw.size() * elvect_size)] = vj_e[k] * rw[i] * t; + coef[k + (i * elvect_size) + (j * rw.size() * elvect_size)] = vj_e[k] * rw[i] * + t; } } } } void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, - std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, - CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - const int rank, Vector &res) + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, + CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, + CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, + const int rank, Vector &res) { const int rxdim = V_x.numColumns(); @@ -2187,10 +2202,13 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } } -void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, - std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, - const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - Vector const &coef, Vector const &DS_coef, const int rank, Vector &res) +void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace + *fesR, + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, + const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, + CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, + Vector const &coef, Vector const &DS_coef, const int rank, Vector &res) { const int rxdim = V_x.numColumns(); const int rvdim = V_v.numColumns(); @@ -2304,7 +2322,8 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fes temp = 0.0; for (int k = 0; k < elvect.Size(); k++) { - temp += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * elvect[k]; + temp += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * + elvect[k]; } res[j] += temp; } @@ -2415,7 +2434,8 @@ void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, res -= rhs_Gw; const double relNorm = res.norm() / std::max(normGsol, normRHS); - cout << rank << ": relative residual norm for NNLS solution of Gs = Gw: " << relNorm << endl; + cout << rank << ": relative residual norm for NNLS solution of Gs = Gw: " << + relNorm << endl; } // Compute EQP solution from constraints on snapshots. @@ -2434,7 +2454,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const int nsnap = BX_snapshots->numColumns(); MFEM_VERIFY(nsnap == BX_snapshots->numColumns() || - nsnap + nsets == BX_snapshots->numColumns(), // Q: nsets? + nsnap + nsets == BX_snapshots->numColumns(), // Q: nsets? ""); MFEM_VERIFY(BV->numRows() == BX_snapshots->numRows(), ""); MFEM_VERIFY(BV->numRows() == fespace_X->GetTrueVSize(), ""); @@ -2660,7 +2680,8 @@ void save_CAROM_Vector(const CAROM::Vector &vector, const std::string &filename) } else { - std::cerr << "Error: Unable to open file " << filename << " for writing." << std::endl; + std::cerr << "Error: Unable to open file " << filename << " for writing." << + std::endl; } } diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index f44621773..a65aab1ea 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -55,22 +55,26 @@ class HyperelasticOperator : public TimeDependentOperator // Compute coefficients of the reduced integrator with respect to inputs Q and x // in HyperelasticNLFIntegrator_ComputeReducedEQP. void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, - std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, - CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef); + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, + CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef); // Perform hyperreduction with EQP void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, - std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, - CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - const int rank, Vector &res); + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, + CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, + CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, + const int rank, Vector &res); // Optimized EQP hyperreduction routine with preallocated arrays -void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace *fesR, - std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, - const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - Vector const &coef, Vector const &DS_coef, const int rank, Vector &res); +void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace + *fesR, + std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, + const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, + CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, + Vector const &coef, Vector const &DS_coef, const int rank, Vector &res); // Compute a row in the G matrix which corresponds to a given FE element void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, @@ -100,7 +104,8 @@ void get_window_ids(int n_step, int n_window, CAROM::Vector *ids); bool fileExists(const std::string &filename); // Helper function to save a CAROM vector to an external file -void save_CAROM_Vector(const CAROM::Vector &vector, const std::string &filename); +void save_CAROM_Vector(const CAROM::Vector &vector, + const std::string &filename); // Helper function to load a CAROM vector from an external file void load_CAROM_vector(const std::string &filename, CAROM::Vector &vector); From 1b80cd9dc364aa7cfab91bad8ac1f35456cdf7b5 Mon Sep 17 00:00:00 2001 From: Dylan Copeland Date: Tue, 24 Oct 2023 17:51:44 -0700 Subject: [PATCH 094/120] Astyle 3.1. --- .../prom/nonlinear_elasticity_global_rom.cpp | 18 +++++++++--------- lib/linalg/Matrix.h | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 64981f354..690e84c0f 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -738,9 +738,9 @@ int main(int argc, char *argv[]) Vector *x_rec = new Vector(x_gf.GetTrueVector()); CAROM::Vector *v_rec_librom = new CAROM::Vector(v_rec->GetData(), v_rec->Size(), - true, false); + true, false); CAROM::Vector *x_rec_librom = new CAROM::Vector(x_rec->GetData(), x_rec->Size(), - true, false); + true, false); // 9. Initialize the hyperelastic operator, the GLVis visualization and print // the initial energies. @@ -816,14 +816,14 @@ int main(int argc, char *argv[]) if (x_base_only == false) { basis_generator_v = new CAROM::BasisGenerator(options, isIncremental, - basisFileName + "_V"); + basisFileName + "_V"); } basis_generator_x = new CAROM::BasisGenerator(options, isIncremental, - basisFileName + "_X"); + basisFileName + "_X"); basis_generator_H = new CAROM::BasisGenerator(options, isIncremental, - basisFileName + "_H"); + basisFileName + "_H"); } RomOperator *romop = 0; @@ -842,7 +842,7 @@ int main(int argc, char *argv[]) CAROM::Vector *window_ids = nullptr; CAROM::Vector *load_eqpsol = new CAROM::Vector(1, - false); // Will be resized later + false); // Will be resized later // 11. Initialize ROM operator // I guess most of this should be done on id =0 @@ -910,7 +910,7 @@ int main(int argc, char *argv[]) // Setup hyperreduction, using either EQP or sampled DOFs and a sample mesh. ParFiniteElementSpace *sp_XV_space; CAROM::Matrix *Hsinv = new CAROM::Matrix(hdim, hdim, - false); // Gets resized before use. + false); // Gets resized before use. const IntegrationRule *ir0 = NULL; if (ir0 == NULL) @@ -1657,7 +1657,7 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, S_hat_v0 = new CAROM::Vector(rvdim, false); S_hat_v0_temp = new Vector(v0_fom.Size()); S_hat_v0_temp_librom = new CAROM::Vector(S_hat_v0_temp->GetData(), - S_hat_v0_temp->Size(), true, false); + S_hat_v0_temp->Size(), true, false); M_hat = new CAROM::Matrix(rvdim, rvdim, false); M_hat_inv = new CAROM::Matrix(rvdim, rvdim, false); @@ -2057,7 +2057,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, for (int k = 0; k < elvect_size; k++) { coef[k + (i * elvect_size) + (j * rw.size() * elvect_size)] = vj_e[k] * rw[i] * - t; + t; } } } diff --git a/lib/linalg/Matrix.h b/lib/linalg/Matrix.h index bf68b33b8..4f7f74aba 100644 --- a/lib/linalg/Matrix.h +++ b/lib/linalg/Matrix.h @@ -946,7 +946,7 @@ class Matrix void orthogonalize(); - /** + /** * @brief Rescale every matrix row by its maximum absolute value. */ void From 4ec599ad2a174ed4bd29d34dfc394563c46d0eb1 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:20:38 -0500 Subject: [PATCH 095/120] updated .gitignore --- .gitignore | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 4c981ce76..279d709c0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,11 +28,8 @@ docs/html # Do not track results regression_tests/results -init_bash.sh .gitignore -.vscode/settings.json -.vscode/libROM.code-workspace -init_bash_debug_arm.sh -# Don't track .txt files -*.txt + +# astyle +astyle From 06d46c10099b43c89e7ecdf66e03d5ab1eec005f Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:28:19 -0500 Subject: [PATCH 096/120] ++ increment --- examples/prom/nonlinear_elasticity_global_rom.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 690e84c0f..f8a98ae1b 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1197,7 +1197,7 @@ int main(int argc, char *argv[]) get_EQPsol(current_window, load_eqpsol); romop->SetEQP(load_eqpsol); ode_solver->Init(*romop); - current_window += 1; + current_window++; } solveTimer.Start(); ode_solver->Step(*wMFEM, t, dt_real); From 08330741e545667b90c405e38392bcc43da52760 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:29:13 -0500 Subject: [PATCH 097/120] Removed ReducedSystemOperator --- examples/prom/nonlinear_elasticity_global_rom.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index f8a98ae1b..f5c26cc4f 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -128,7 +128,6 @@ using namespace mfem; #include "nonlinear_elasticity_global_rom_eqp.hpp" -class ReducedSystemOperator; class RomOperator : public TimeDependentOperator { From 6c25d124f06abaf8f5c9c916c141aa640341c385 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:31:17 -0500 Subject: [PATCH 098/120] Removed unnecessary comments --- examples/prom/nonlinear_elasticity_global_rom.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index f5c26cc4f..62bbbcc81 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -61,7 +61,7 @@ // Relative error of ROM velocity (v) at t_final: 5 is 0.978726 // // Online phase with EQP hyper-reduction -// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -ntw 50 -rvdim 40 -rxdim 10 -hdim 1 -sc 1.00 +// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -ntw 50 -rvdim 40 -rxdim 10 -hdim 1 -sc 1.00 // Output message: // Elapsed time for time integration loop 82.0641 // Relative error of ROM position (x) at t_final: 5 is 0.000893109 @@ -2004,7 +2004,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, int index = 0; eprev = -1; // For every quadrature weight - for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 + for (int i = 0; i < rw.size(); ++i) { const int e = qp[i] / nqe; // Element index // Local (element) index of the quadrature point From baed4e566dff2c8d9c0f9d0038862ad9c332be28 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:32:52 -0500 Subject: [PATCH 099/120] Added consts --- examples/prom/nonlinear_elasticity_global_rom.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 62bbbcc81..a8fab209a 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2027,7 +2027,7 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, eltrans->SetIntPoint(&ip); model->SetTransformation(*eltrans); // Get the transformation weight - double t = eltrans->Weight(); + const double t = eltrans->Weight(); // Calculate DS and store CalcInverse(eltrans->Jacobian(), Jrt); @@ -2334,8 +2334,8 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, FiniteElement const &fe, ElementTransformation &Trans, Vector &r) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); - int dof = fe.GetDof(); // Get number of dofs in element - int dim = fe.GetDim(); + const int dof = fe.GetDof(); // Get number of dofs in element + const int dim = fe.GetDim(); // Initialize nonlinear operator matrices (there is probably a better way) DenseMatrix DSh(dof, dim); From b930f52ddcdffc61390f26e3325efa627698dab8 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:34:45 -0500 Subject: [PATCH 100/120] Added more ++ increments --- examples/prom/nonlinear_elasticity_global_rom.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index a8fab209a..16dd42acc 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1086,7 +1086,7 @@ int main(int argc, char *argv[]) { if (Ess_mat_sp(i, 0) == 1) { - num_ess_sp += 1; + num_ess_sp++; } } @@ -1100,7 +1100,7 @@ int main(int argc, char *argv[]) if (Ess_mat_sp(i, 0) == 1) { ess_tdof_list_sp[ctr] = i; - ctr += 1; + ctr++; } } } @@ -2640,18 +2640,18 @@ void get_window_ids(int n_step, int n_window, CAROM::Vector *ids) { if (i <= res - res_up) { - ctr += 1; + ctr++; } if (i > 2 && i <= res_up + 1) { - ctr += 1; + ctr++; } } else { if (i <= res + 1) { - ctr += 1; + ctr++; } } ids->item(i - 1) += ctr; From 9bc906c6a75a9347591df1f8e2b154e31780380f Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:38:35 -0500 Subject: [PATCH 101/120] Removed unnecessary HSINV --- examples/prom/nonlinear_elasticity_global_rom.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 16dd42acc..f9b918489 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -908,8 +908,6 @@ int main(int argc, char *argv[]) // Setup hyperreduction, using either EQP or sampled DOFs and a sample mesh. ParFiniteElementSpace *sp_XV_space; - CAROM::Matrix *Hsinv = new CAROM::Matrix(hdim, hdim, - false); // Gets resized before use. const IntegrationRule *ir0 = NULL; if (ir0 == NULL) From 141f46c29c5faf2c7a8620dead452c12be328f0d Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:52:54 -0500 Subject: [PATCH 102/120] Comments --- examples/prom/nonlinear_elasticity_global_rom.cpp | 4 +++- examples/prom/nonlinear_elasticity_global_rom_eqp.hpp | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index f9b918489..26db333ba 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2451,7 +2451,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const int nsnap = BX_snapshots->numColumns(); MFEM_VERIFY(nsnap == BX_snapshots->numColumns() || - nsnap + nsets == BX_snapshots->numColumns(), // Q: nsets? + nsnap + nsets == BX_snapshots->numColumns(), ""); MFEM_VERIFY(BV->numRows() == BX_snapshots->numRows(), ""); MFEM_VERIFY(BV->numRows() == fespace_X->GetTrueVSize(), ""); @@ -2502,6 +2502,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } // Outer loop for time windowing + // NB: Currently time windowing is done using the same velocity basis for every time window. + // A possible next step is to also do time windowing on the velocity basis, which should lead to smaller EQP systems. for (int oi = 0; oi < outer_loop_length; ++oi) { if (window_ids) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index a65aab1ea..169fbee3e 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -10,7 +10,6 @@ // Functions used by nonlinear_elastic_global_rom.cpp with EQP. #include "mfem/Utilities.hpp" -// #include "fem/nonlininteg.hpp" #include using namespace mfem; using namespace std; From b72df7dba6846e540f5939bd5327c2c3e4df3ba5 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 09:05:24 -0500 Subject: [PATCH 103/120] LibROM native read / write --- .../prom/nonlinear_elasticity_global_rom.cpp | 54 ++----------------- 1 file changed, 3 insertions(+), 51 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 26db333ba..3453bf9e9 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -128,7 +128,6 @@ using namespace mfem; #include "nonlinear_elasticity_global_rom_eqp.hpp" - class RomOperator : public TimeDependentOperator { private: @@ -2594,7 +2593,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, SolveNNLS(rank, nnls_tol, maxNNLSnnz, w, Gt, sol); if (window_ids) { - save_CAROM_Vector(sol, "sol_" + std::to_string(oi) + ".csv"); + sol.write("sol_" + std::to_string(oi)); } } } @@ -2665,58 +2664,11 @@ bool fileExists(const std::string &filename) return file.good(); } -void save_CAROM_Vector(const CAROM::Vector &vector, const std::string &filename) -{ - std::ofstream file(filename); - if (file.is_open()) - { - for (int i = 0; i < vector.dim(); ++i) - { - file << vector(i) << "\n"; - } - file.close(); - std::cout << "Vector saved as " << filename << " successfully." << std::endl; - } - else - { - std::cerr << "Error: Unable to open file " << filename << " for writing." << - std::endl; - } -} - -void load_CAROM_vector(const std::string &filename, CAROM::Vector &vector) -{ - std::ifstream file(filename); - std::string line; - std::vector data; - - while (std::getline(file, line)) - { - std::stringstream ss(line); - std::string value; - - while (std::getline(ss, value, ',')) - { - data.push_back(std::stod(value)); - } - } - - // Set the size of the vector - int size = data.size(); - vector.setSize(size); - - // Copy the data into the vector - for (int i = 0; i < size; ++i) - { - vector(i) = data[i]; - } -} - void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol) { - string filename = "sol_" + std::to_string(current_window) + ".csv"; + string filename = "sol_" + std::to_string(current_window); if (fileExists(filename)) { - load_CAROM_vector(filename, *load_eqpsol); + load_eqpsol.read(filename); } } From e5bda39964da91701fd41b4dc85f9b1858c83c71 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 09:14:26 -0500 Subject: [PATCH 104/120] Added pointer deletion --- examples/prom/nonlinear_elasticity_global_rom.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 3453bf9e9..b25d683d7 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1423,6 +1423,16 @@ int main(int argc, char *argv[]) delete romop; delete soper; + delete BV_librom; + delete BX_librom; + delete H_librom; + delete Hsinv; + delete smm; + delete eqpSol; + delete eqpSol_S; + delete window_ids; + delete load_eqpsol; + totalTimer.Stop(); if (myid == 0) cout << "Elapsed time for entire simulation " << totalTimer.RealTime() << endl; @@ -2242,6 +2252,8 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace Vector Vx_e; // Lift x, add x0 and prolongate result + // NB: We are lifting x to the FOM space. + // This could be further optimized by using a reduced mesh consisting of only the elements containing EQP points. V_x.mult(x, Vx_librom_temp); add(*Vx_temp, *x0, Vx); P->Mult(Vx, p_Vx); From 8997e888e45c29babe5c34c882295efa577deb8a Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 10:54:29 -0500 Subject: [PATCH 105/120] librom runs --- .../prom/nonlinear_elasticity_global_rom.cpp | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index b25d683d7..eb3568dff 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -38,34 +38,36 @@ // // Create FOM comparison data: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -id 2 +// Output message: +// Elapsed time for time integration loop 10.6499 // // Online phase with full sampling: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rvdim 40 -rxdim 10 -hdim 71 -nsr 1170 -sc 1.0 // Output message: -// Elapsed time for time integration loop 1.80759 +// Elapsed time for time integration loop 9.76384 // Relative error of ROM position (x) at t_final: 5 is 0.000231698 // Relative error of ROM velocity (v) at t_final: 5 is 0.466941 // // Online phase with strong hyper-reduction, using GNAT (over-sampled DEIM): // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rvdim 40 -rxdim 10 -hdim 71 -nsr 100 -sc 1.0 // Output message: -// Elapsed time for time integration loop 1.0111 +// Elapsed time for time integration loop 5.50959 // Relative error of ROM position (x) at t_final: 5 is 0.00209877 // Relative error of ROM velocity (v) at t_final: 5 is 1.39472 // // Online phase with strong hyper-reduction, using QDEIM: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype qdeim -rvdim 40 -rxdim 10 -hdim 71 -nsr 100 -sc 1.0 // Output message: -// Elapsed time for time integration loop 1.02559 +// Elapsed time for time integration loop 5.65571 // Relative error of ROM position (x) at t_final: 5 is 0.00188458 // Relative error of ROM velocity (v) at t_final: 5 is 0.978726 // // Online phase with EQP hyper-reduction -// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -ntw 50 -rvdim 40 -rxdim 10 -hdim 1 -sc 1.00 +// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -ntw 50 -rvdim 40 -rxdim 10 -hdim 1 -sc 1.0 // Output message: -// Elapsed time for time integration loop 82.0641 -// Relative error of ROM position (x) at t_final: 5 is 0.000893109 -// Relative error of ROM velocity (v) at t_final: 5 is 0.741266 +// Elapsed time for time integration loop 0.248581 +// Relative error of ROM position (x) at t_final: 5 is 0.0034452 +// Relative error of ROM velocity (v) at t_final: 5 is 1.55409 // // ================================================================================= // @@ -80,6 +82,8 @@ // // Create FOM comparison data: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -xbo -def-ic -id 2 +// Output message: +// Elapsed time for time integration loop 11.0755 // // Online phase with full sampling: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rxdim 57 -hdim 183 -nsr 1170 -sc 1.0 -xbo -def-ic @@ -104,9 +108,9 @@ // // Online phase with EQP hyper reduction: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -rxdim 2 -hdim 1 -ntw 25 -sc 1.00 -xbo -def-ic -// Elapsed time for time integration loop 0.766614 -// Relative error of ROM position (x) at t_final: 5 is 0.0161132 -// Relative error of ROM velocity (v) at t_final: 5 is 0.775545 +// Elapsed time for time integration loop 0.0496174 +// Relative error of ROM position (x) at t_final: 5 is 0.0248637 +// Relative error of ROM velocity (v) at t_final: 5 is 1 // This example runs in parallel with MPI, by using the same number of MPI ranks // in all phases (offline, merge, online). @@ -896,7 +900,7 @@ int main(int argc, char *argv[]) { hdim = H_librom->numColumns(); } - + CAROM::Matrix* Hsinv = new CAROM::Matrix(hdim, hdim, false); MFEM_VERIFY(H_librom->numColumns() >= hdim, ""); if (H_librom->numColumns() > hdim) @@ -949,7 +953,7 @@ int main(int argc, char *argv[]) nsamp_H = hdim; } - CAROM::Matrix* Hsinv = new CAROM::Matrix(nsamp_H, hdim, false); + Hsinv->setSize(nsamp_H, hdim); vector sample_dofs(nsamp_H); // Setup hyperreduction using DEIM, GNAT, or S-OPT @@ -2681,6 +2685,6 @@ void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol) string filename = "sol_" + std::to_string(current_window); if (fileExists(filename)) { - load_eqpsol.read(filename); + load_eqpsol->read(filename); } } From 4acb4e88b33fa1fc9c9ce8f86dd54b2be3f6c0b1 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 11:13:46 -0500 Subject: [PATCH 106/120] New benchmarks --- examples/prom/nonlinear_elasticity_global_rom.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index eb3568dff..782cfb47b 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -88,27 +88,27 @@ // Online phase with full sampling: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rxdim 57 -hdim 183 -nsr 1170 -sc 1.0 -xbo -def-ic // Output message: -// Elapsed time for time integration loop 18.9874 +// Elapsed time for time integration loop 12.814 // Relative error of ROM position (x) at t_final: 5 is 7.08272e-05 // Relative error of ROM velocity (v) at t_final: 5 is 0.00387647 // // Online phase with strong hyper reduction, using GNAT (over-sampled DEIM): // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rxdim 2 -hdim 4 -nsr 10 -sc 1.0 -xbo -def-ic // Output message: -// Elapsed time for time integration loop 0.120194 +// Elapsed time for time integration loop 0.651751 // Relative error of ROM position (x) at t_final: 5 is 0.0130818 // Relative error of ROM velocity (v) at t_final: 5 is 0.979978 // // Online phase with strong hyper reduction, using QDEIM: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype qdeim -rxdim 2 -hdim 4 -nsr 10 -sc 1.0 -xbo -def-ic // Output message: -// Elapsed time for time integration loop 0.10806 +// Elapsed time for time integration loop 0.655331 // Relative error of ROM position (x) at t_final: 5 is 0.0108709 // Relative error of ROM velocity (v) at t_final: 5 is 1.30704 // // Online phase with EQP hyper reduction: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -rxdim 2 -hdim 1 -ntw 25 -sc 1.00 -xbo -def-ic -// Elapsed time for time integration loop 0.0496174 +// Elapsed time for time integration loop 0.0638885 // Relative error of ROM position (x) at t_final: 5 is 0.0248637 // Relative error of ROM velocity (v) at t_final: 5 is 1 // This example runs in parallel with MPI, by using the same number of MPI ranks From ff390688ea13143633ef1df33e2a87cedffd8945 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 11:54:03 -0500 Subject: [PATCH 107/120] elementmatrix first implementation --- .../prom/nonlinear_elasticity_global_rom.cpp | 64 ++++++++++++++----- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 782cfb47b..a8d219422 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -247,6 +247,32 @@ class ElasticEnergyCoefficient : public Coefficient virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip); virtual ~ElasticEnergyCoefficient() {} }; +struct ElemMatrices { + DenseMatrix DSh; + DenseMatrix DS; + DenseMatrix Jrt; + DenseMatrix Jpt; + DenseMatrix P; + DenseMatrix PMatI; + DenseMatrix PMatO; + Vector elvect; + // Constructor for matrices struct + ElemMatrices(int dof, int dim) : + DSh(dof, dim), + DS(dof, dim), + Jrt(dim), + Jpt(dim), + P(dim), + PMatI(), + PMatO(), + elvect(dof * dim) + { + // Set dimensions for PMatI and PMatO + //PMatI.UseExternalData(elfun.GetData(), dof, dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); + } + +}; void InitialDeformationIC1(const Vector &x, Vector &y); @@ -1957,6 +1983,7 @@ void RomOperator::SetEQP(CAROM::Vector *eqpSol) ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef); } + // Functions for EQP functionality // Compute coefficients of the reduced integrator with respect to inputs Q and x @@ -2344,14 +2371,15 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &ve_j, NeoHookeanModel *model, Vector const &elfun, - FiniteElement const &fe, ElementTransformation &Trans, Vector &r) + FiniteElement const &fe, ElementTransformation &Trans, Vector &r, const int dof, const int dim, + ElemMatrices &em) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); - const int dof = fe.GetDof(); // Get number of dofs in element - const int dim = fe.GetDim(); + /* const int dof = fe.GetDof(); // Get number of dofs in element + const int dim = fe.GetDim(); */ // Initialize nonlinear operator matrices (there is probably a better way) - DenseMatrix DSh(dof, dim); + /* DenseMatrix DSh(dof, dim); DenseMatrix DS(dof, dim); DenseMatrix Jrt(dim); DenseMatrix Jpt(dim); @@ -2360,14 +2388,15 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, PMatI.UseExternalData(elfun.GetData(), dof, dim); DenseMatrix PMatO; Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); */ +em.PMatI.UseExternalData(elfun.GetData(), dof, dim); model->SetTransformation(Trans); // For each integration point for (int i = 0; i < ir->GetNPoints(); i++) { - elvect = 0.0; + em.elvect = 0.0; // Get integration point const IntegrationPoint &ip = ir->IntPoint(i); @@ -2378,20 +2407,20 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, double t = Trans.Weight(); // Compute action of nonlinear operator - CalcInverse(Trans.Jacobian(), Jrt); - fe.CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P); - P *= t; // NB: Not by ip.weight - AddMultABt(DS, P, PMatO); + CalcInverse(Trans.Jacobian(), em.Jrt); + fe.CalcDShape(ip, em.DSh); + Mult(em.DSh, em.Jrt, em.DS); + MultAtB(em.PMatI, em.DS, em.Jpt); + model->EvalP(em.Jpt, em.P); + em.P *= t; // NB: Not by ip.weight + AddMultABt(em.DS, em.P, em.PMatO); r[i] = 0.0; // Calculate r[i] = ve_j^T * elvect - for (int k = 0; k < elvect.Size(); k++) + for (int k = 0; k < em.elvect.Size(); k++) { - r[i] += ve_j[k] * elvect[k]; + r[i] += ve_j[k] * em.elvect[k]; } } } @@ -2579,7 +2608,10 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, P->Mult(v_j, pv_j); pv_j.GetSubVector(vdofs, ve_j); // Compute the row of G corresponding to element e, store in r - ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r); + const int dof = fe.GetDof(); // Get number of dofs in element + const int dim = fe.GetDim(); + ElemMatrices em(dof, dim); + ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r, dof, dim, em); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; From 3d888717fd7dc4ac4677fc63585035f6f5257cf1 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Wed, 29 Nov 2023 12:00:46 -0500 Subject: [PATCH 108/120] Construct em outside loop --- .../prom/nonlinear_elasticity_global_rom.cpp | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index a8d219422..76a3619af 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2375,20 +2375,6 @@ void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, ElemMatrices &em) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); - /* const int dof = fe.GetDof(); // Get number of dofs in element - const int dim = fe.GetDim(); */ - - // Initialize nonlinear operator matrices (there is probably a better way) - /* DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DenseMatrix Jpt(dim); - DenseMatrix P(dim); - DenseMatrix PMatI; // Extract element dofs - PMatI.UseExternalData(elfun.GetData(), dof, dim); - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); */ em.PMatI.UseExternalData(elfun.GetData(), dof, dim); model->SetTransformation(Trans); @@ -2560,6 +2546,13 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, Gt.setSize(NQ, NB * current_size); } + + // Assuming dof and dim are constant, initialize element matrices once + const FiniteElement &fe1 = *fespace_X->GetFE(0); + const int dof = fe1.GetDof(); + const int dim = fe1.GetDim(); + ElemMatrices em(dof, dim); + // For every snapshot in batch for (int i = i_start; i < i_end; ++i) { @@ -2585,6 +2578,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, Array vdofs; DofTransformation *doftrans = fespace_X->GetElementVDofs(e, vdofs); const FiniteElement &fe = *fespace_X->GetFE(e); + ElementTransformation *eltrans = fespace_X->GetElementTransformation(e); px_i.GetSubVector(vdofs, elfun); @@ -2608,9 +2602,6 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, P->Mult(v_j, pv_j); pv_j.GetSubVector(vdofs, ve_j); // Compute the row of G corresponding to element e, store in r - const int dof = fe.GetDof(); // Get number of dofs in element - const int dim = fe.GetDim(); - ElemMatrices em(dof, dim); ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r, dof, dim, em); for (int m = 0; m < nqe; ++m) From 1adfb9c5f83437c020b47b344f6831ad969e3666 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 30 Nov 2023 19:50:44 -0500 Subject: [PATCH 109/120] Added element matrices to ROM class --- .../prom/nonlinear_elasticity_global_rom.cpp | 48 +++++-------------- .../nonlinear_elasticity_global_rom_eqp.hpp | 38 +++++++++++---- 2 files changed, 41 insertions(+), 45 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 76a3619af..d733da363 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -186,6 +186,7 @@ class RomOperator : public TimeDependentOperator Vector eqp_coef; Vector eqp_DS_coef; const bool fastIntegration = true; + ElemMatrices *em; int rank; @@ -247,32 +248,6 @@ class ElasticEnergyCoefficient : public Coefficient virtual double Eval(ElementTransformation &T, const IntegrationPoint &ip); virtual ~ElasticEnergyCoefficient() {} }; -struct ElemMatrices { - DenseMatrix DSh; - DenseMatrix DS; - DenseMatrix Jrt; - DenseMatrix Jpt; - DenseMatrix P; - DenseMatrix PMatI; - DenseMatrix PMatO; - Vector elvect; - // Constructor for matrices struct - ElemMatrices(int dof, int dim) : - DSh(dof, dim), - DS(dof, dim), - Jrt(dim), - Jpt(dim), - P(dim), - PMatI(), - PMatO(), - elvect(dof * dim) - { - // Set dimensions for PMatI and PMatO - //PMatI.UseExternalData(elfun.GetData(), dof, dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); - } - -}; void InitialDeformationIC1(const Vector &x, Vector &y); @@ -1805,9 +1780,13 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, cout << myid << ": EQP using " << elements.size() << " elements out of " << fom->fespace.GetNE() << endl; + const FiniteElement &fe1 = *(fom->fespace).GetFE(0); + const int dof = fe1.GetDof(); + const int dim = fe1.GetDim(); + em = new ElemMatrices(dof,dim); GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, - ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef); + ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef, em); } } @@ -1980,7 +1959,7 @@ void RomOperator::SetEQP(CAROM::Vector *eqpSol) << fom->fespace.GetNE() << endl; GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, - ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef); + ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef, em); } @@ -1991,7 +1970,7 @@ void RomOperator::SetEQP(CAROM::Vector *eqpSol) void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, - CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef) + CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef, ElemMatrices *em) { const int rvdim = V_v.numColumns(); const int fomdim = V_v.numRows(); @@ -2034,9 +2013,6 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, const FiniteElement *fe = fesR->GetFE(0); dof = fe->GetDof(); dim = fe->GetDim(); - DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); DS_coef.SetSize(dof * dim * rw.size()); DS_coef = 0.0; int index = 0; @@ -2068,15 +2044,15 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, const double t = eltrans->Weight(); // Calculate DS and store - CalcInverse(eltrans->Jacobian(), Jrt); - fe->CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); + CalcInverse(eltrans->Jacobian(), em->Jrt); + fe->CalcDShape(ip, em->DSh); + Mult(em->DSh, em->Jrt, em->DS); for (int ii = 0; ii < dof; ++ii) { for (int jj = 0; jj < dim; ++jj) { index = jj + ii * dim; - DS_coef[index + (i * dof * dim)] = DS.Elem(ii, jj); + DS_coef[index + (i * dof * dim)] = em->DS.Elem(ii, jj); } } // For every basis vector diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 169fbee3e..6f2cf99de 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -51,12 +51,38 @@ class HyperelasticOperator : public TimeDependentOperator virtual ~HyperelasticOperator(); }; + +struct ElemMatrices +{ + DenseMatrix DSh; + DenseMatrix DS; + DenseMatrix Jrt; + DenseMatrix Jpt; + DenseMatrix P; + DenseMatrix PMatI; + DenseMatrix PMatO; + Vector elvect; + // Constructor for matrices struct + ElemMatrices(int dof, int dim) : DSh(dof, dim), + DS(dof, dim), + Jrt(dim), + Jpt(dim), + P(dim), + PMatI(), + PMatO(), + elvect(dof * dim) + { + // Set dimensions for PMatI and PMatO + PMatO.UseExternalData(elvect.GetData(), dof, dim); + } +}; + // Compute coefficients of the reduced integrator with respect to inputs Q and x // in HyperelasticNLFIntegrator_ComputeReducedEQP. void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, - CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef); + CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef, ElemMatrices *em); // Perform hyperreduction with EQP void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, @@ -78,7 +104,8 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace // Compute a row in the G matrix which corresponds to a given FE element void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &ve_j, NeoHookeanModel *model, Vector const &elfun, - FiniteElement const &fe, ElementTransformation &Trans, Vector &r); + FiniteElement const &fe, ElementTransformation &Trans, Vector &r, const int dof, const int dim, + ElemMatrices &em); void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, CAROM::Vector const &w, CAROM::Matrix &Gt, @@ -102,12 +129,5 @@ void get_window_ids(int n_step, int n_window, CAROM::Vector *ids); // Helper function to check if a file exists bool fileExists(const std::string &filename); -// Helper function to save a CAROM vector to an external file -void save_CAROM_Vector(const CAROM::Vector &vector, - const std::string &filename); - -// Helper function to load a CAROM vector from an external file -void load_CAROM_vector(const std::string &filename, CAROM::Vector &vector); - // Load a EQP solution void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol); From 87a8e4131fe5db100430e253a1211c63da498dcb Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 30 Nov 2023 20:12:08 -0500 Subject: [PATCH 110/120] Reduced EQP fast uses em --- .../prom/nonlinear_elasticity_global_rom.cpp | 31 ++++++++----------- .../nonlinear_elasticity_global_rom_eqp.hpp | 4 ++- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index d733da363..716253399 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -1821,7 +1821,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, model, x0, V_x, V_v, x_librom, Vx_librom_temp, Vx_temp, - eqp_coef, eqp_DS_coef, rank, resEQP); + eqp_coef, eqp_DS_coef, rank, resEQP, em); } else HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, @@ -2221,7 +2221,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - Vector const &coef, Vector const &DS_coef, const int rank, Vector &res) + Vector const &coef, Vector const &DS_coef, const int rank, Vector &res, ElemMatrices *em) { const int rxdim = V_x.numColumns(); const int rvdim = V_v.numColumns(); @@ -2271,13 +2271,6 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace dof = fe->GetDof(); dim = fe->GetDim(); int index = 0; - DenseMatrix DS(dof, dim); - DenseMatrix Jpt(dim); - DenseMatrix P_f(dim); - DenseMatrix PMatI; // Extract element dofs - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); eprev = -1; double temp = 0.0; @@ -2310,35 +2303,37 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace } // Integration at ip - elvect = 0.0; - PMatI.UseExternalData(Vx_e.GetData(), dof, dim); + em->elvect = 0.0; + em->PMatI.UseExternalData(Vx_e.GetData(), dof, dim); // Set integration point in the element transformation eltrans->SetIntPoint(&ip); model->SetTransformation(*eltrans); + const int elvec_size = em->elvect.Size(); for (int ii = 0; ii < dof; ++ii) { for (int jj = 0; jj < dim; ++jj) { index = jj + ii * dim; - DS.Elem(ii, jj) = DS_coef[index + (i * elvect.Size())]; + em->DS.Elem(ii, jj) = DS_coef[index + (i * elvec_size)]; } } - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P_f); - AddMultABt(DS, P_f, PMatO); + MultAtB(em->PMatI, em->DS, em->Jpt); + model->EvalP(em->Jpt, em->P_f); + AddMultABt(em->DS, em->P_f, em->PMatO); // Calculate r[i] = ve_j^T * elvect // coef is size len(vdofs) * rvdim * rw.size + const int qp_size = qp.size(); for (int j = 0; j < rvdim; ++j) { temp = 0.0; - for (int k = 0; k < elvect.Size(); k++) + for (int k = 0; k < elvec_size; k++) { - temp += coef[k + (i * elvect.Size()) + (j * qp.size() * elvect.Size())] * - elvect[k]; + temp += coef[k + (i * elvec_size) + (j * qp_size * elvec_size)] * + em->elvect[k]; } res[j] += temp; } diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 6f2cf99de..3842aa4d8 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -59,6 +59,7 @@ struct ElemMatrices DenseMatrix Jrt; DenseMatrix Jpt; DenseMatrix P; + DenseMatrix P_f; DenseMatrix PMatI; DenseMatrix PMatO; Vector elvect; @@ -68,6 +69,7 @@ struct ElemMatrices Jrt(dim), Jpt(dim), P(dim), + P_f(dim), PMatI(), PMatO(), elvect(dof * dim) @@ -99,7 +101,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - Vector const &coef, Vector const &DS_coef, const int rank, Vector &res); + Vector const &coef, Vector const &DS_coef, const int rank, Vector &res, ElemMatrices *em); // Compute a row in the G matrix which corresponds to a given FE element void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, From fdaf2abda920b89dd6b6021e933ea2c71b5c9188 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 30 Nov 2023 20:42:16 -0500 Subject: [PATCH 111/120] Em on everything --- .../prom/nonlinear_elasticity_global_rom.cpp | 32 +++++++++---------- .../nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 716253399..3720db4c3 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -189,7 +189,7 @@ class RomOperator : public TimeDependentOperator ElemMatrices *em; int rank; - + NeoHookeanModel *model; protected: @@ -1827,7 +1827,7 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, model, x0, V_x, V_v, x_librom, Vx_librom_temp, Vx_temp, - rank, resEQP); + rank, resEQP, em); Vector recv(resEQP); MPI_Allreduce(resEQP.GetData(), recv.GetData(), resEQP.Size(), MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); @@ -2081,7 +2081,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - const int rank, Vector &res) + const int rank, Vector &res, ElemMatrices *em) { const int rxdim = V_x.numColumns(); @@ -2133,7 +2133,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, fe = fesR->GetFE(0); dof = fe->GetDof(); dim = fe->GetDim(); - DenseMatrix DSh(dof, dim); +/* DenseMatrix DSh(dof, dim); DenseMatrix DS(dof, dim); DenseMatrix Jrt(dim); DenseMatrix Jpt(dim); @@ -2141,7 +2141,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, DenseMatrix PMatI; // Extract element dofs DenseMatrix PMatO; Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); */ eprev = -1; double temp = 0.0; @@ -2173,9 +2173,9 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } // Integration at ip - PMatI.UseExternalData(Vx_e.GetData(), dof, dim); + em->PMatI.UseExternalData(Vx_e.GetData(), dof, dim); - elvect = 0.0; + em->elvect = 0.0; // Set integration point in the element transformation eltrans->SetIntPoint(&ip); @@ -2185,13 +2185,13 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, double t = eltrans->Weight(); // Compute action of nonlinear operator - CalcInverse(eltrans->Jacobian(), Jrt); - fe->CalcDShape(ip, DSh); - Mult(DSh, Jrt, DS); - MultAtB(PMatI, DS, Jpt); - model->EvalP(Jpt, P_f); - P_f *= (t * rw[i]); // NB: Not by ip.weight - AddMultABt(DS, P_f, PMatO); + CalcInverse(eltrans->Jacobian(), em->Jrt); + fe->CalcDShape(ip, em->DSh); + Mult(em->DSh, em->Jrt, em->DS); + MultAtB(em->PMatI, em->DS, em->Jpt); + model->EvalP(em->Jpt, em->P_f); + em->P_f *= (t * rw[i]); // NB: Not by ip.weight + AddMultABt(em->DS, em->P_f, em->PMatO); // For every basis vector for (int j = 0; j < rvdim; ++j) @@ -2206,9 +2206,9 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, temp = 0.0; // Calculate r[i] = ve_j^T * elvect - for (int k = 0; k < elvect.Size(); k++) + for (int k = 0; k < em->elvect.Size(); k++) { - temp += vj_e[k] * elvect[k]; + temp += vj_e[k] * em->elvect[k]; } res[j] += temp; } diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 3842aa4d8..005be40e1 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -92,7 +92,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - const int rank, Vector &res); + const int rank, Vector &res, ElemMatrices *em); // Optimized EQP hyperreduction routine with preallocated arrays void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace From b67409408d4e3160e2d5ac2a7a8103e9aedcb603 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 1 Dec 2023 08:49:55 -0500 Subject: [PATCH 112/120] Debugged EQP --- .../prom/nonlinear_elasticity_global_rom.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 3720db4c3..c7f613228 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -65,9 +65,9 @@ // Online phase with EQP hyper-reduction // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -ntw 50 -rvdim 40 -rxdim 10 -hdim 1 -sc 1.0 // Output message: -// Elapsed time for time integration loop 0.248581 -// Relative error of ROM position (x) at t_final: 5 is 0.0034452 -// Relative error of ROM velocity (v) at t_final: 5 is 1.55409 +// Elapsed time for time integration loop 5.50702 +// Relative error of ROM position (x) at t_final: 5 is 0.000498426 +// Relative error of ROM velocity (v) at t_final: 5 is 1.64583 // // ================================================================================= // @@ -108,9 +108,9 @@ // // Online phase with EQP hyper reduction: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -rxdim 2 -hdim 1 -ntw 25 -sc 1.00 -xbo -def-ic -// Elapsed time for time integration loop 0.0638885 -// Relative error of ROM position (x) at t_final: 5 is 0.0248637 -// Relative error of ROM velocity (v) at t_final: 5 is 1 +// Elapsed time for time integration loop 0.202143 +// Relative error of ROM position (x) at t_final: 5 is 0.0189361 +// Relative error of ROM velocity (v) at t_final: 5 is 0.830724 // This example runs in parallel with MPI, by using the same number of MPI ranks // in all phases (offline, merge, online). @@ -2677,8 +2677,10 @@ bool fileExists(const std::string &filename) void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol) { string filename = "sol_" + std::to_string(current_window); - if (fileExists(filename)) + if (fileExists(filename+".000000")) { + std::cout << "The length of the vector is: " << load_eqpsol->dim() << std::endl; load_eqpsol->read(filename); } + } From 127e6cd780473709a85c7f5f5b4710228e055f3b Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:07:01 -0500 Subject: [PATCH 113/120] Formatting --- .../prom/nonlinear_elasticity_global_rom.cpp | 54 ++++++++++--------- .../nonlinear_elasticity_global_rom_eqp.hpp | 27 +++++----- 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index c7f613228..9a825a427 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -189,7 +189,7 @@ class RomOperator : public TimeDependentOperator ElemMatrices *em; int rank; - + NeoHookeanModel *model; protected: @@ -1781,9 +1781,9 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, cout << myid << ": EQP using " << elements.size() << " elements out of " << fom->fespace.GetNE() << endl; const FiniteElement &fe1 = *(fom->fespace).GetFE(0); - const int dof = fe1.GetDof(); - const int dim = fe1.GetDim(); - em = new ElemMatrices(dof,dim); + const int dof = fe1.GetDof(); + const int dim = fe1.GetDim(); + em = new ElemMatrices(dof,dim); GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef, em); @@ -1970,7 +1970,8 @@ void RomOperator::SetEQP(CAROM::Vector *eqpSol) void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, - CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef, ElemMatrices *em) + CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef, + ElemMatrices *em) { const int rvdim = V_v.numColumns(); const int fomdim = V_v.numRows(); @@ -2133,15 +2134,15 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, fe = fesR->GetFE(0); dof = fe->GetDof(); dim = fe->GetDim(); -/* DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DenseMatrix Jpt(dim); - DenseMatrix P_f(dim); - DenseMatrix PMatI; // Extract element dofs - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); */ + /* DenseMatrix DSh(dof, dim); + DenseMatrix DS(dof, dim); + DenseMatrix Jrt(dim); + DenseMatrix Jpt(dim); + DenseMatrix P_f(dim); + DenseMatrix PMatI; // Extract element dofs + DenseMatrix PMatO; + Vector elvect(dof * dim); + PMatO.UseExternalData(elvect.GetData(), dof, dim); */ eprev = -1; double temp = 0.0; @@ -2221,7 +2222,8 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - Vector const &coef, Vector const &DS_coef, const int rank, Vector &res, ElemMatrices *em) + Vector const &coef, Vector const &DS_coef, const int rank, Vector &res, + ElemMatrices *em) { const int rxdim = V_x.numColumns(); const int rvdim = V_v.numColumns(); @@ -2342,12 +2344,13 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &ve_j, NeoHookeanModel *model, Vector const &elfun, - FiniteElement const &fe, ElementTransformation &Trans, Vector &r, const int dof, const int dim, + FiniteElement const &fe, ElementTransformation &Trans, Vector &r, const int dof, + const int dim, ElemMatrices &em) { MFEM_VERIFY(r.Size() == ir->GetNPoints(), ""); -em.PMatI.UseExternalData(elfun.GetData(), dof, dim); + em.PMatI.UseExternalData(elfun.GetData(), dof, dim); model->SetTransformation(Trans); // For each integration point @@ -2452,7 +2455,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const int nsnap = BX_snapshots->numColumns(); MFEM_VERIFY(nsnap == BX_snapshots->numColumns() || - nsnap + nsets == BX_snapshots->numColumns(), + nsnap + nsets == BX_snapshots->numColumns(), ""); MFEM_VERIFY(BV->numRows() == BX_snapshots->numRows(), ""); MFEM_VERIFY(BV->numRows() == fespace_X->GetTrueVSize(), ""); @@ -2518,11 +2521,11 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, } - // Assuming dof and dim are constant, initialize element matrices once + // Assuming dof and dim are constant, initialize element matrices once const FiniteElement &fe1 = *fespace_X->GetFE(0); - const int dof = fe1.GetDof(); - const int dim = fe1.GetDim(); - ElemMatrices em(dof, dim); + const int dof = fe1.GetDof(); + const int dim = fe1.GetDim(); + ElemMatrices em(dof, dim); // For every snapshot in batch for (int i = i_start; i < i_end; ++i) @@ -2549,7 +2552,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, Array vdofs; DofTransformation *doftrans = fespace_X->GetElementVDofs(e, vdofs); const FiniteElement &fe = *fespace_X->GetFE(e); - + ElementTransformation *eltrans = fespace_X->GetElementTransformation(e); px_i.GetSubVector(vdofs, elfun); @@ -2573,7 +2576,8 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, P->Mult(v_j, pv_j); pv_j.GetSubVector(vdofs, ve_j); // Compute the row of G corresponding to element e, store in r - ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r, dof, dim, em); + ComputeElementRowOfG(ir0, vdofs, ve_j, model, elfun, fe, *eltrans, r, dof, dim, + em); for (int m = 0; m < nqe; ++m) Gt((e * nqe) + m, j + (i * NB)) = r[m]; @@ -2682,5 +2686,5 @@ void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol) std::cout << "The length of the vector is: " << load_eqpsol->dim() << std::endl; load_eqpsol->read(filename); } - + } diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index 005be40e1..fd3c0ad8e 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -65,14 +65,14 @@ struct ElemMatrices Vector elvect; // Constructor for matrices struct ElemMatrices(int dof, int dim) : DSh(dof, dim), - DS(dof, dim), - Jrt(dim), - Jpt(dim), - P(dim), - P_f(dim), - PMatI(), - PMatO(), - elvect(dof * dim) + DS(dof, dim), + Jrt(dim), + Jpt(dim), + P(dim), + P_f(dim), + PMatI(), + PMatO(), + elvect(dof * dim) { // Set dimensions for PMatI and PMatO PMatO.UseExternalData(elvect.GetData(), dof, dim); @@ -84,7 +84,8 @@ struct ElemMatrices void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, - CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef, ElemMatrices *em); + CAROM::Matrix const &V_v, const int rank, Vector &coef, Vector &DS_coef, + ElemMatrices *em); // Perform hyperreduction with EQP void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, @@ -101,13 +102,15 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - Vector const &coef, Vector const &DS_coef, const int rank, Vector &res, ElemMatrices *em); + Vector const &coef, Vector const &DS_coef, const int rank, Vector &res, + ElemMatrices *em); // Compute a row in the G matrix which corresponds to a given FE element void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs, Vector const &ve_j, NeoHookeanModel *model, Vector const &elfun, - FiniteElement const &fe, ElementTransformation &Trans, Vector &r, const int dof, const int dim, - ElemMatrices &em); + FiniteElement const &fe, ElementTransformation &Trans, Vector &r, const int dof, + const int dim, + ElemMatrices &em); void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, CAROM::Vector const &w, CAROM::Matrix &Gt, From 65cdd110516cfa881247a99719d2f1151b3bf708 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 1 Dec 2023 16:56:39 -0500 Subject: [PATCH 114/120] Removed unnecessary comment --- examples/prom/nonlinear_elasticity_global_rom.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 9a825a427..5f071009a 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2147,7 +2147,7 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, eprev = -1; double temp = 0.0; // For every quadrature weight - for (int i = 0; i < rw.size(); ++i) // NOTE: i < 9 + for (int i = 0; i < rw.size(); ++i) { const int e = qp[i] / nqe; // Element index // Local (element) index of the quadrature point From 1651898d19be9396fda95fb5e53b85f3dd6c9579 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 1 Dec 2023 17:12:00 -0500 Subject: [PATCH 115/120] Removed more unnecessary comments --- examples/prom/nonlinear_elasticity_global_rom.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 5f071009a..c0fe751fc 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -2134,15 +2134,6 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, fe = fesR->GetFE(0); dof = fe->GetDof(); dim = fe->GetDim(); - /* DenseMatrix DSh(dof, dim); - DenseMatrix DS(dof, dim); - DenseMatrix Jrt(dim); - DenseMatrix Jpt(dim); - DenseMatrix P_f(dim); - DenseMatrix PMatI; // Extract element dofs - DenseMatrix PMatO; - Vector elvect(dof * dim); - PMatO.UseExternalData(elvect.GetData(), dof, dim); */ eprev = -1; double temp = 0.0; From e445e53c147681bd468de09192af0adef9afbe20 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 7 Dec 2023 19:08:57 -0500 Subject: [PATCH 116/120] Implemented subsampling --- .../prom/nonlinear_elasticity_global_rom.cpp | 34 +++++++++++++++---- .../nonlinear_elasticity_global_rom_eqp.hpp | 2 +- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index c0fe751fc..809300adc 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -69,6 +69,13 @@ // Relative error of ROM position (x) at t_final: 5 is 0.000498426 // Relative error of ROM velocity (v) at t_final: 5 is 1.64583 // +// Online phase with EQP hyper-reduction and subsampling of snapshots +// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -ntw 50 -rvdim 40 -rxdim 10 -hdim 1 -sc 1.0 -sbs 10 +// Output message: +// Elapsed time for time integration loop 0.614632 +// Relative error of ROM position (x) at t_final: 5 is 0.00247479 +// Relative error of ROM velocity (v) at t_final: 5 is 1.33349 +// // ================================================================================= // // Sample runs and results for parametric ROM using only displacement basis @@ -465,6 +472,9 @@ int main(int argc, char *argv[]) // Number of time windows int n_windows = 0; + // Subsampling steps + int snap_step = 1; + OptionsParser args(argc, argv); args.AddOption(&mesh_file, "-m", "--mesh", "Mesh file to use."); @@ -539,6 +549,8 @@ int main(int argc, char *argv[]) "Maximum nnz for NNLS"); args.AddOption(&n_windows, "-ntw", "--n-time-windows", "Number of time windows. Default is 0"); + args.AddOption(&snap_step, "-sbs", "--subsampling-step", + "Subsamble every n-th snapshot. Default is 1, meaning no subsampling"); args.Parse(); if (!args.Good()) @@ -936,7 +948,7 @@ int main(int argc, char *argv[]) SetupEQP_snapshots(ir0, myid, &fespace, nsets, BV_librom, GetSnapshotMatrix(fespace.GetTrueVSize(), nsets, max_num_snapshots, "X"), vx0.GetBlock(0), - preconditionNNLS, tolNNLS, maxNNLSnnz, model, *eqpSol, window_ids); + preconditionNNLS, tolNNLS, maxNNLSnnz, model, *eqpSol, window_ids, snap_step); if (writeSampleMesh) WriteMeshEQP(pmesh, myid, ir0->GetNPoints(), *eqpSol); @@ -2430,6 +2442,17 @@ void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, relNorm << endl; } +int getSteps(const int nsnap, const int snap_step) +{ if (nsnap % snap_step != 0.0) + { + return int(nsnap / snap_step) + 1; + } + else + { + return nsnap / snap_step; + } +} + // Compute EQP solution from constraints on snapshots. void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, ParFiniteElementSpace *fespace_X, @@ -2437,7 +2460,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const CAROM::Matrix *BX_snapshots, const Vector x0, const bool precondition, const double nnls_tol, const int maxNNLSnnz, NeoHookeanModel *model, - CAROM::Vector &sol, CAROM::Vector *window_ids) + CAROM::Vector &sol, CAROM::Vector *window_ids, const int snap_step) { const int nqe = ir0->GetNPoints(); const int ne = fespace_X->GetNE(); @@ -2454,7 +2477,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const bool skipFirstW = (nsnap + nsets == BX_snapshots->numColumns()); // Compute G of size (NB * nsnap) x NQ, but only store its transpose Gt. - CAROM::Matrix Gt(NQ, NB * nsnap, true); + CAROM::Matrix Gt(NQ, NB * getSteps(nsnap, snap_step), true); int current_size = 0; int i_start = 0; int i_end = 0; @@ -2508,10 +2531,9 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, current_size = i_end - i_start; cout << "i_start = " << i_start << endl; cout << "Number of NNLS equations: " << NB * current_size << endl; - Gt.setSize(NQ, NB * current_size); + Gt.setSize(NQ, NB * getSteps(current_size, snap_step)); } - // Assuming dof and dim are constant, initialize element matrices once const FiniteElement &fe1 = *fespace_X->GetFE(0); const int dof = fe1.GetDof(); @@ -2519,7 +2541,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, ElemMatrices em(dof, dim); // For every snapshot in batch - for (int i = i_start; i < i_end; ++i) + for (int i = i_start; i < i_end; i += snap_step) { // Set the sampled dofs from the snapshot matrix for (int j = 0; j < BX_snapshots->numRows(); ++j) diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index fd3c0ad8e..fa4ea2ddd 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -123,7 +123,7 @@ void SetupEQP_snapshots(const IntegrationRule *ir0, const int rank, const CAROM::Matrix *BX_snapshots, const Vector x0, const bool precondition, const double nnls_tol, const int maxNNLSnnz, NeoHookeanModel *model, - CAROM::Vector &sol, CAROM::Vector *window_ids); + CAROM::Vector &sol, CAROM::Vector *window_ids, const int snap_step); void WriteMeshEQP(ParMesh *pmesh, const int myid, const int nqe, CAROM::Vector const &eqpSol); From db3dfdbc5c5c5a9a46a3ccaae1e511163d5e9d9d Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 7 Dec 2023 19:30:02 -0500 Subject: [PATCH 117/120] measure elapsed time for EQP --- examples/prom/nonlinear_elasticity_global_rom.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index 809300adc..ad370d3c6 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -68,6 +68,7 @@ // Elapsed time for time integration loop 5.50702 // Relative error of ROM position (x) at t_final: 5 is 0.000498426 // Relative error of ROM velocity (v) at t_final: 5 is 1.64583 +// Elapsed time for entire simulation xxx // // Online phase with EQP hyper-reduction and subsampling of snapshots // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -ntw 50 -rvdim 40 -rxdim 10 -hdim 1 -sc 1.0 -sbs 10 @@ -75,6 +76,7 @@ // Elapsed time for time integration loop 0.614632 // Relative error of ROM position (x) at t_final: 5 is 0.00247479 // Relative error of ROM velocity (v) at t_final: 5 is 1.33349 +// Elapsed time for entire simulation 11.2825 // // ================================================================================= // From cc6b4c7eb71c637e1ce03724de9fa41b14c980e5 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Thu, 7 Dec 2023 20:26:12 -0500 Subject: [PATCH 118/120] Measure elapsed time for all methods --- .../prom/nonlinear_elasticity_global_rom.cpp | 47 +++++++++++++------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index ad370d3c6..a30c13b50 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -39,36 +39,40 @@ // Create FOM comparison data: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -id 2 // Output message: -// Elapsed time for time integration loop 10.6499 +// Elapsed time for time integration loop 9.17153 +// Elapsed time for entire simulation 10.0001 // // Online phase with full sampling: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rvdim 40 -rxdim 10 -hdim 71 -nsr 1170 -sc 1.0 // Output message: -// Elapsed time for time integration loop 9.76384 +// Elapsed time for time integration loop 7.03877 // Relative error of ROM position (x) at t_final: 5 is 0.000231698 // Relative error of ROM velocity (v) at t_final: 5 is 0.466941 +// Elapsed time for entire simulation 8.98334 // // Online phase with strong hyper-reduction, using GNAT (over-sampled DEIM): // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rvdim 40 -rxdim 10 -hdim 71 -nsr 100 -sc 1.0 // Output message: -// Elapsed time for time integration loop 5.50959 +// Elapsed time for time integration loop 4.18114 // Relative error of ROM position (x) at t_final: 5 is 0.00209877 // Relative error of ROM velocity (v) at t_final: 5 is 1.39472 +// Elapsed time for entire simulation 4.52802 // // Online phase with strong hyper-reduction, using QDEIM: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype qdeim -rvdim 40 -rxdim 10 -hdim 71 -nsr 100 -sc 1.0 // Output message: -// Elapsed time for time integration loop 5.65571 +// Elapsed time for time integration loop 4.2972 // Relative error of ROM position (x) at t_final: 5 is 0.00188458 // Relative error of ROM velocity (v) at t_final: 5 is 0.978726 +// Elapsed time for entire simulation 5.33154 // // Online phase with EQP hyper-reduction // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -ntw 50 -rvdim 40 -rxdim 10 -hdim 1 -sc 1.0 // Output message: -// Elapsed time for time integration loop 5.50702 -// Relative error of ROM position (x) at t_final: 5 is 0.000498426 -// Relative error of ROM velocity (v) at t_final: 5 is 1.64583 -// Elapsed time for entire simulation xxx +// Elapsed time for time integration loop 2.70006 +// Relative error of ROM position (x) at t_final: 5 is 0.00169666 +// Relative error of ROM velocity (v) at t_final: 5 is 17.71 +// Elapsed time for entire simulation 831.216 // // Online phase with EQP hyper-reduction and subsampling of snapshots // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -ntw 50 -rvdim 40 -rxdim 10 -hdim 1 -sc 1.0 -sbs 10 @@ -92,34 +96,47 @@ // Create FOM comparison data: // ./nonlinear_elasticity_global_rom -offline -dt 0.01 -tf 5.0 -s 14 -vs 100 -sc 1.0 -xbo -def-ic -id 2 // Output message: -// Elapsed time for time integration loop 11.0755 +// Elapsed time for time integration loop 9.61062 +// Elapsed time for entire simulation 10.3806 // // Online phase with full sampling: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rxdim 57 -hdim 183 -nsr 1170 -sc 1.0 -xbo -def-ic // Output message: -// Elapsed time for time integration loop 12.814 +// Elapsed time for time integration loop 8.43858 // Relative error of ROM position (x) at t_final: 5 is 7.08272e-05 // Relative error of ROM velocity (v) at t_final: 5 is 0.00387647 +// Elapsed time for entire simulation 24.7245 // // Online phase with strong hyper reduction, using GNAT (over-sampled DEIM): // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype gnat -rxdim 2 -hdim 4 -nsr 10 -sc 1.0 -xbo -def-ic // Output message: -// Elapsed time for time integration loop 0.651751 +// Elapsed time for time integration loop 0.507339 // Relative error of ROM position (x) at t_final: 5 is 0.0130818 // Relative error of ROM velocity (v) at t_final: 5 is 0.979978 +// Elapsed time for entire simulation 0.583618 // // Online phase with strong hyper reduction, using QDEIM: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -hyp -hrtype qdeim -rxdim 2 -hdim 4 -nsr 10 -sc 1.0 -xbo -def-ic // Output message: -// Elapsed time for time integration loop 0.655331 -// Relative error of ROM position (x) at t_final: 5 is 0.0108709 -// Relative error of ROM velocity (v) at t_final: 5 is 1.30704 +// Elapsed time for time integration loop 0.504657 +// Relative error of ROM position (x) at t_final: 5 is 0.00883079 +// Relative error of ROM velocity (v) at t_final: 5 is 1.26721 +// Elapsed time for entire simulation 0.581446 // // Online phase with EQP hyper reduction: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -rxdim 2 -hdim 1 -ntw 25 -sc 1.00 -xbo -def-ic -// Elapsed time for time integration loop 0.202143 +// Elapsed time for time integration loop 0.184455 // Relative error of ROM position (x) at t_final: 5 is 0.0189361 // Relative error of ROM velocity (v) at t_final: 5 is 0.830724 +// Elapsed time for entire simulation 5.05504 +// +// Online phase with EQP hyper reduction and subsampling of snapshots: +// ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -rxdim 2 -hdim 1 -ntw 25 -sc 1.00 -xbo -def-ic -sbs 10 +// Elapsed time for time integration loop 0.0882588 +// Relative error of ROM position (x) at t_final: 5 is 0.0243952 +// Relative error of ROM velocity (v) at t_final: 5 is 0.986434 +// Elapsed time for entire simulation 0.868528 +// // This example runs in parallel with MPI, by using the same number of MPI ranks // in all phases (offline, merge, online). From 396d31146ba4c4224520613ffafc976793dc01a1 Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Sat, 9 Dec 2023 05:55:07 +0100 Subject: [PATCH 119/120] Update .gitignore --- .gitignore | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.gitignore b/.gitignore index 279d709c0..016626da8 100644 --- a/.gitignore +++ b/.gitignore @@ -28,8 +28,3 @@ docs/html # Do not track results regression_tests/results -.gitignore - - -# astyle -astyle From e2093a3b8b8b289197444d5467ff8ca8413143ce Mon Sep 17 00:00:00 2001 From: Axel Larsson <65452706+axla-io@users.noreply.github.com> Date: Fri, 22 Dec 2023 16:19:44 +0100 Subject: [PATCH 120/120] Optimized lifting of FOM dimension in Nonlinear elasticity EQP (#259) * implementation works and gives speed up * Removed unnecessary variables * Final benchmark --- .../prom/nonlinear_elasticity_global_rom.cpp | 224 ++++++++++++------ .../nonlinear_elasticity_global_rom_eqp.hpp | 19 +- 2 files changed, 154 insertions(+), 89 deletions(-) diff --git a/examples/prom/nonlinear_elasticity_global_rom.cpp b/examples/prom/nonlinear_elasticity_global_rom.cpp index a30c13b50..7936d143c 100644 --- a/examples/prom/nonlinear_elasticity_global_rom.cpp +++ b/examples/prom/nonlinear_elasticity_global_rom.cpp @@ -77,10 +77,10 @@ // Online phase with EQP hyper-reduction and subsampling of snapshots // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -ntw 50 -rvdim 40 -rxdim 10 -hdim 1 -sc 1.0 -sbs 10 // Output message: -// Elapsed time for time integration loop 0.614632 +// Elapsed time for time integration loop 0.552546 // Relative error of ROM position (x) at t_final: 5 is 0.00247479 // Relative error of ROM velocity (v) at t_final: 5 is 1.33349 -// Elapsed time for entire simulation 11.2825 +// Elapsed time for entire simulation 11.469 // // ================================================================================= // @@ -125,17 +125,17 @@ // // Online phase with EQP hyper reduction: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -rxdim 2 -hdim 1 -ntw 25 -sc 1.00 -xbo -def-ic -// Elapsed time for time integration loop 0.184455 +// Elapsed time for time integration loop 0.144368 // Relative error of ROM position (x) at t_final: 5 is 0.0189361 // Relative error of ROM velocity (v) at t_final: 5 is 0.830724 -// Elapsed time for entire simulation 5.05504 +// Elapsed time for entire simulation 4.97996 // // Online phase with EQP hyper reduction and subsampling of snapshots: // ./nonlinear_elasticity_global_rom -online -dt 0.01 -tf 5.0 -s 14 -vs 100 -eqp -ns 2 -rxdim 2 -hdim 1 -ntw 25 -sc 1.00 -xbo -def-ic -sbs 10 -// Elapsed time for time integration loop 0.0882588 +// Elapsed time for time integration loop 0.0141128 // Relative error of ROM position (x) at t_final: 5 is 0.0243952 // Relative error of ROM velocity (v) at t_final: 5 is 0.986434 -// Elapsed time for entire simulation 0.868528 +// Elapsed time for entire simulation 0.782108 // // This example runs in parallel with MPI, by using the same number of MPI ranks // in all phases (offline, merge, online). @@ -192,12 +192,12 @@ class RomOperator : public TimeDependentOperator bool x_base_only; CAROM::Vector *pfom_librom, *pfom_v_librom; - Vector* pfom; - Vector* pfom_x; - Vector* pfom_v; - mutable Vector* zfom_x; - mutable Vector* zfom_v; - CAROM::Vector* zfom_x_librom; + Vector *pfom; + Vector *pfom_x; + Vector *pfom_v; + mutable Vector *zfom_x; + mutable Vector *zfom_v; + CAROM::Vector *zfom_x_librom; CAROM::SampleMeshManager *smm; @@ -214,6 +214,10 @@ class RomOperator : public TimeDependentOperator const bool fastIntegration = true; ElemMatrices *em; + CAROM::Matrix eqp_lifting; + std::vector eqp_liftDOFs; + mutable CAROM::Vector eqp_lifted; + int rank; NeoHookeanModel *model; @@ -303,8 +307,8 @@ CAROM::Matrix *GetFirstColumns(const int N, const CAROM::Matrix *A) } // TODO: move this to the library? -void BasisGeneratorFinalSummary(CAROM::BasisGenerator* bg, - const double energyFraction, int& cutoff, +void BasisGeneratorFinalSummary(CAROM::BasisGenerator *bg, + const double energyFraction, int &cutoff, const std::string cutoffOutputPath) { const int rom_dim = bg->getSpatialBasis()->numColumns(); @@ -932,7 +936,7 @@ int main(int argc, char *argv[]) { hdim = H_librom->numColumns(); } - CAROM::Matrix* Hsinv = new CAROM::Matrix(hdim, hdim, false); + CAROM::Matrix *Hsinv = new CAROM::Matrix(hdim, hdim, false); MFEM_VERIFY(H_librom->numColumns() >= hdim, ""); if (H_librom->numColumns() > hdim) @@ -1814,10 +1818,49 @@ RomOperator::RomOperator(HyperelasticOperator *fom_, const FiniteElement &fe1 = *(fom->fespace).GetFE(0); const int dof = fe1.GetDof(); const int dim = fe1.GetDim(); - em = new ElemMatrices(dof,dim); + em = new ElemMatrices(dof, dim); GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef, em); + + // Set the matrix of the linear map from the ROM X space to all DOFs on + // sampled elements. + Array vdofs; + int numSampledDofs = 0; + for (auto e : elements) + { + fom->fespace.GetElementVDofs(e, vdofs); + numSampledDofs += vdofs.Size(); + } + + eqp_lifting.setSize(numSampledDofs, rxdim); + eqp_liftDOFs.resize(numSampledDofs); + eqp_lifted.setSize(numSampledDofs); + + int cnt = 0; + for (auto e : elements) + { + fom->fespace.GetElementVDofs(e, vdofs); + for (int i = 0; i < vdofs.Size(); ++i, ++cnt) + eqp_liftDOFs[cnt] = + vdofs[i]; // eqp_liftDOFs maps an index of eqp_lifted to the FOM dof index + } + + MFEM_VERIFY(cnt == numSampledDofs, ""); + + CAROM::Vector ej(rxdim, false); + + for (int j = 0; j < rxdim; ++j) + { + ej = 0.0; + ej(j) = 1.0; + V_x.mult(ej, *zfom_x_librom); + + for (int i = 0; i < numSampledDofs; ++i) + { + eqp_lifting(i, j) = (*zfom_x)[eqp_liftDOFs[i]]; + } + } } } @@ -1850,15 +1893,13 @@ void RomOperator::Mult_Hyperreduced(const Vector &vx, Vector &dvx_dt) const if (fastIntegration) { HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(&(fom->fespace), eqp_rw, - eqp_qp, ir_eqp, model, - x0, V_x, V_v, x_librom, Vx_librom_temp, Vx_temp, - eqp_coef, eqp_DS_coef, rank, resEQP, em); + eqp_qp, ir_eqp, model, x0, rvdim, x_librom,eqp_coef, eqp_DS_coef, rank, resEQP, + em, eqp_lifting, eqp_liftDOFs, eqp_lifted); } else - HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, - eqp_qp, ir_eqp, model, x0, - V_x, V_v, x_librom, Vx_librom_temp, Vx_temp, - rank, resEQP, em); + HyperelasticNLFIntegrator_ComputeReducedEQP(&(fom->fespace), eqp_rw, eqp_qp, + ir_eqp, model, x0, V_v, x_librom, rank, resEQP, em, eqp_lifting, eqp_liftDOFs, + eqp_lifted); Vector recv(resEQP); MPI_Allreduce(resEQP.GetData(), recv.GetData(), resEQP.Size(), MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); @@ -1991,8 +2032,46 @@ void RomOperator::SetEQP(CAROM::Vector *eqpSol) GetEQPCoefficients_HyperelasticNLFIntegrator(&(fom->fespace), eqp_rw, eqp_qp, ir_eqp, model, V_v, rank, eqp_coef, eqp_DS_coef, em); -} + // Set the matrix of the linear map from the ROM X space to all DOFs on + // sampled elements. + Array vdofs; + int numSampledDofs = 0; + for (auto e : elements) + { + fom->fespace.GetElementVDofs(e, vdofs); + numSampledDofs += vdofs.Size(); + } + + eqp_lifting.setSize(numSampledDofs, rxdim); + eqp_liftDOFs.resize(numSampledDofs); + eqp_lifted.setSize(numSampledDofs); + + int cnt = 0; + for (auto e : elements) + { + fom->fespace.GetElementVDofs(e, vdofs); + for (int i = 0; i < vdofs.Size(); ++i, ++cnt) + eqp_liftDOFs[cnt] = + vdofs[i]; // eqp_liftDOFs maps an index of eqp_lifted to the FOM dof index + } + + MFEM_VERIFY(cnt == numSampledDofs, ""); + + CAROM::Vector ej(rxdim, false); + + for (int j = 0; j < rxdim; ++j) + { + ej = 0.0; + ej(j) = 1.0; + V_x.mult(ej, *zfom_x_librom); + + for (int i = 0; i < numSampledDofs; ++i) + { + eqp_lifting(i, j) = (*zfom_x)[eqp_liftDOFs[i]]; + } + } +} // Functions for EQP functionality @@ -2111,17 +2190,14 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, - CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, - CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - const int rank, Vector &res, ElemMatrices *em) + CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res, + ElemMatrices *em, const CAROM::Matrix eqp_lifting, + const std::vector eqp_liftDOFs,CAROM::Vector eqp_lifted) { - const int rxdim = V_x.numColumns(); const int rvdim = V_v.numColumns(); const int fomdim = V_v.numRows(); MFEM_VERIFY(rw.size() == qp.size(), ""); - MFEM_VERIFY(x.dim() == rxdim, ""); - MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); MFEM_VERIFY(rank == 0, "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); @@ -2140,34 +2216,29 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, int dof = 0; int dim = 0; - // Get prolongation matrix - const Operator *P = fesR->GetProlongationMatrix(); - - // Vectors to be prolongated - Vector Vx(fomdim); + // Basis vector Vector vj(fomdim); - // Prolongated vectors - Vector p_Vx(P->Height()); - Vector p_vj(P->Height()); - // Element vectors Vector Vx_e; Vector vj_e; - // Lift x, add x0 and prolongate result - V_x.mult(x, Vx_librom_temp); - add(*Vx_temp, *x0, Vx); - P->Mult(Vx, p_Vx); + // Lift x, add x0 + eqp_lifting.mult(x, eqp_lifted); + + for (int i = 0; i < eqp_liftDOFs.size(); ++i) + eqp_lifted(i) += x0->Elem(eqp_liftDOFs[i]); // Initialize nonlinear operator storage // Assuming all elements have the same dim and n_dof fe = fesR->GetFE(0); dof = fe->GetDof(); dim = fe->GetDim(); + Vx_e.SetSize(dof * dim); eprev = -1; double temp = 0.0; + int eqp_ctr = 0; // For every quadrature weight for (int i = 0; i < rw.size(); ++i) { @@ -2191,7 +2262,11 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } // Get element vectors - p_Vx.GetSubVector(vdofs, Vx_e); + for (int i = 0; i < dof * dim; ++i) + { + Vx_e.Elem(i) = eqp_lifted(eqp_ctr * dof * dim + i); + } + eqp_ctr++; eprev = e; } @@ -2222,9 +2297,8 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, // Get basis vector and prolongate for (int k = 0; k < V_v.numRows(); ++k) vj[k] = V_v(k, j); - P->Mult(vj, p_vj); - p_vj.GetSubVector(vdofs, vj_e); + vj.GetSubVector(vdofs, vj_e); temp = 0.0; @@ -2239,22 +2313,17 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, } void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace - *fesR, - std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, - const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, - CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - Vector const &coef, Vector const &DS_coef, const int rank, Vector &res, - ElemMatrices *em) + *fesR,std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, + const int rvdim, + CAROM::Vector const &x, Vector const &coef, Vector const &DS_coef, + const int rank, Vector &res, + ElemMatrices *em, const CAROM::Matrix eqp_lifting, + const std::vector eqp_liftDOFs,CAROM::Vector eqp_lifted) { - const int rxdim = V_x.numColumns(); - const int rvdim = V_v.numColumns(); - const int fomdim = V_x.numRows(); - MFEM_VERIFY(x.dim() == rxdim, ""); - MFEM_VERIFY(V_x.numRows() == fesR->GetTrueVSize(), ""); MFEM_VERIFY(rank == 0, - "TODO: generalize to parallel. This uses full dofs in V, which has true dofs"); + "TODO: generalize to parallel. This uses full dofs in X, which has true dofs"); const int nqe = ir->GetWeights().Size(); @@ -2270,37 +2339,29 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace int dof = 0; int dim = 0; - // Get prolongation matrix - const Operator *P = fesR->GetProlongationMatrix(); - - // Vectors to be prolongated - Vector Vx(fomdim); - - // Prolongated vectors - Vector p_Vx(P->Height()); - // Element vectors Vector Vx_e; - // Lift x, add x0 and prolongate result - // NB: We are lifting x to the FOM space. - // This could be further optimized by using a reduced mesh consisting of only the elements containing EQP points. - V_x.mult(x, Vx_librom_temp); - add(*Vx_temp, *x0, Vx); - P->Mult(Vx, p_Vx); + // Lift x, add x0 + eqp_lifting.mult(x, eqp_lifted); + + for (int i = 0; i < eqp_liftDOFs.size(); ++i) + eqp_lifted(i) += x0->Elem(eqp_liftDOFs[i]); // Initialize nonlinear operator storage // Assuming all elements have the same dim and n_dof fe = fesR->GetFE(0); dof = fe->GetDof(); dim = fe->GetDim(); + Vx_e.SetSize(dof * dim); int index = 0; eprev = -1; double temp = 0.0; + int eqp_ctr = 0; // For every quadrature weight - for (int i = 0; i < qp.size(); ++i) // NOTE: i < 9 + for (int i = 0; i < qp.size(); ++i) { const int e = qp[i] / nqe; // Element index // Local (element) index of the quadrature point @@ -2322,7 +2383,12 @@ void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace } // Get element vectors - p_Vx.GetSubVector(vdofs, Vx_e); + for (int i = 0; i < dof * dim; ++i) + { + Vx_e.Elem(i) = eqp_lifted(eqp_ctr * dof * dim + i); + } + eqp_ctr++; + eprev = e; } @@ -2462,7 +2528,8 @@ void SolveNNLS(const int rank, const double nnls_tol, const int maxNNLSnnz, } int getSteps(const int nsnap, const int snap_step) -{ if (nsnap % snap_step != 0.0) +{ + if (nsnap % snap_step != 0.0) { return int(nsnap / snap_step) + 1; } @@ -2713,10 +2780,9 @@ bool fileExists(const std::string &filename) void get_EQPsol(const int current_window, CAROM::Vector *load_eqpsol) { string filename = "sol_" + std::to_string(current_window); - if (fileExists(filename+".000000")) + if (fileExists(filename + ".000000")) { std::cout << "The length of the vector is: " << load_eqpsol->dim() << std::endl; load_eqpsol->read(filename); } - } diff --git a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp index fa4ea2ddd..5cd4ef1d6 100644 --- a/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp +++ b/examples/prom/nonlinear_elasticity_global_rom_eqp.hpp @@ -91,19 +91,18 @@ void GetEQPCoefficients_HyperelasticNLFIntegrator(ParFiniteElementSpace *fesR, void HyperelasticNLFIntegrator_ComputeReducedEQP(ParFiniteElementSpace *fesR, std::vector const &rw, std::vector const &qp, const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, - CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, CAROM::Vector const &x, - CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - const int rank, Vector &res, ElemMatrices *em); + CAROM::Matrix const &V_v, CAROM::Vector const &x, const int rank, Vector &res, + ElemMatrices *em, const CAROM::Matrix eqp_lifting, + const std::vector eqp_liftDOFs,CAROM::Vector eqp_lifted); // Optimized EQP hyperreduction routine with preallocated arrays void HyperelasticNLFIntegrator_ComputeReducedEQP_Fast(ParFiniteElementSpace - *fesR, - std::vector const &rw, std::vector const &qp, - const IntegrationRule *ir, NeoHookeanModel *model, - const Vector *x0, CAROM::Matrix const &V_x, CAROM::Matrix const &V_v, - CAROM::Vector const &x, CAROM::Vector *Vx_librom_temp, Vector *Vx_temp, - Vector const &coef, Vector const &DS_coef, const int rank, Vector &res, - ElemMatrices *em); + *fesR,std::vector const &rw, std::vector const &qp, + const IntegrationRule *ir, NeoHookeanModel *model, const Vector *x0, + const int rvdim, CAROM::Vector const &x, Vector const &coef, + Vector const &DS_coef, const int rank, Vector &res, ElemMatrices *em, + const CAROM::Matrix eqp_lifting, const std::vector eqp_liftDOFs, + CAROM::Vector eqp_lifted); // Compute a row in the G matrix which corresponds to a given FE element void ComputeElementRowOfG(const IntegrationRule *ir, Array const &vdofs,