diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000..743a8ef78 --- /dev/null +++ b/.clang-format @@ -0,0 +1,14 @@ +--- +BasedOnStyle: LLVM +UseTab: Never +IndentWidth: 3 +TabWidth: 3 +BreakBeforeBraces: Attach +PointerAlignment: Left +AlignAfterOpenBracket: true +AllowShortIfStatementsOnASingleLine: false +IndentCaseLabels: false +ColumnLimit: 120 +AccessModifierOffset: -3 +... + diff --git a/.gitignore b/.gitignore index 41e06d5d2..2445ec181 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ *.vlsv *.silo *.o +vscode/ +.vscode/ version.cpp \#*\# *~ diff --git a/.vimrc b/.vimrc new file mode 100644 index 000000000..458e90675 --- /dev/null +++ b/.vimrc @@ -0,0 +1,6 @@ +set cindent +set shiftwidth=3 +set tabstop=3 +set expandtab + +set equalprg=clang-format\ -style=file diff --git a/MAKE/Makefile.Freezer b/MAKE/Makefile.Freezer new file mode 100644 index 000000000..fceda1478 --- /dev/null +++ b/MAKE/Makefile.Freezer @@ -0,0 +1,106 @@ +CMP = mpic++ +LNK = mpic++ + +# Markus' desktop computer + +#======== Vectorization ========== +#Set vector backend type for vlasov solvers, sets precision and length. +#Options: +# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER +# AVX512: VEC8D_AGNER, VEC16F_AGNER +# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK + +ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) +#Single-precision + VECTORCLASS = VEC8F_AGNER + INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass +# VECTORCLASS = VEC8F_FALLBACK +# INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/../vlasiator/vlasovsolver +else +#Double-precision +# VECTORCLASS = VEC4D_AGNER +# INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass + VECTORCLASS = VEC4D_FALLBACK + INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/../vlasiator/vlasovsolver +endif + +#FLAGS = -ggdb + +#GNU flags: +CC_BRAND = gcc +CC_BRAND_VERSION = 6.2.0 +CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -fabi-version=0 -mavx2 +#CXXFLAGS += -ggdb -O0 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -fabi-version=0 -mavx2 +testpackage: CXXFLAGS = -g -ggdb -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 -mno-avx -mno-fma -fno-unsafe-math-optimizations + +MATHFLAGS = -ffast-math +testpackage: MATHFLAGS = -fno-unsafe-math-optimizations + +LDFLAGS = +#-g -ggdb +LIB_MPI = -lgomp -lgfortran -lpapi + + +#======== PAPI ========== +#Add PAPI_MEM define to use papi to report memory consumption? +#CXXFLAGS += -DPAPI_MEM +#testpackage: CXXFLAGS += -DPAPI_MEM + +#======== Allocator ========= +#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc +#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it +#CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +#testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE + + +#======= Compiler and compilation flags ========= +# NOTES on compiler flags: +# CXXFLAGS is for compiler flags, they are always used +# MATHFLAGS are for special math etc. flags, these are only applied on solver functions +# LDFLAGS flags for linker + +# BOOST_VERSION = current trilinos version +# ZOLTAN_VERSION = current trilinos verson + +#======== Libraries =========== + +MPT_VERSION = 7.5.1 +JEMALLOC_VERSION = 4.0.4 +LIBRARY_PREFIX = /home/markusb/git/vlasiator-lib + + +#compiled libraries +# INC_BOOST = -I$(CRAY_TRILINOS_PREFIX_DIR)/include/boost +# LIB_BOOST = -L$(CRAY_TRILINOS_PREFIX_DIR)/lib -lboost_program_options + +# INC_ZOLTAN = -I$(CRAY_TRILINOS_PREFIX_DIR)/include +# LIB_ZOLTAN = -I$(CRAY_TRILINOS_PREFIX_DIR)/lib -lzoltan + +INC_BOOST = -I/usr/include/trilinos +LIB_BOOST = -L/usr/include/trilinos -lboost_program_options + +INC_ZOLTAN = -I/usr/include/trilinos +#LIB_ZOLTAN = -I/usr/lib/x86_64-linux-gnu -ltrilinos_zoltan +LIB_ZOLTAN = -ltrilinos_zoltan +USE_TRILINOS=1 + +INC_JEMALLOC = -I$(LIBRARY_PREFIX)/jemalloc/include +LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/jemalloc/lib -ljemalloc -Wl,-rpath=$(LIBRARY_PREFIX)/jemalloc/lib + + +INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv +LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -Wl,-rpath=$(LIBRARY_PREFIX)/vlsv/lib + +LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib +INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include + +#header libraries + +INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen +#INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen-eigen-07105f7124f9 +INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg_new_neighbours +#INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg_master +INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid + + + diff --git a/MAKE/Makefile.abel_gcc b/MAKE/Makefile.abel_gcc deleted file mode 100644 index 834d46c72..000000000 --- a/MAKE/Makefile.abel_gcc +++ /dev/null @@ -1,62 +0,0 @@ -CMP = mpic++ -LNK = mpic++ - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -#CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = gcc -CC_VERSION = 4.7.2 -CXXFLAGS += -O4 -fopenmp -funroll-loops -std=c++0x -W -Wall -pedantic -Wno-unused -DEIGEN_NO_DEBUG -fabi-version=6 -# CXXFLAGS += -g -O0 -funroll-loops -fopenmp -std=c++0x -fabi-version=6 -mfma4 -MATHFLAGS = -ffast-math -LDFLAGS = -LIB_MPI = -lgomp - -#BOOST_VERSION = 1.53.0 -MPI_BRAND = openmpi -MPI_VERSION = 1.7.3 -#ZOLTAN_VERSION = 3.6 -#SILO_VERSION = 4.8 - -LIBRARY_PREFIX_A = /usit/abel/u1/pr2f2813/libraries -LIBRARY_PREFIX_B = /cluster/software/VERSIONS - - - -INC_BOOST = -I$(LIBRARY_PREFIX_B)/boost-1.53.0/include -LIB_BOOST = -L$(LIBRARY_PREFIX_B)/boost-1.53.0/lib -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX_A)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_VERSION)/zoltan/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX_A)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_VERSION)/zoltan/lib -lzoltan - -#INC_SILO = -I$(LIBRARY_PREFIX_A)/cray-mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/include -#LIB_SILO = -L$(LIBRARY_PREFIX_A)/cray-mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/lib -lsilo - - -#Official dccrg installation -INC_DCCRG = -I$(LIBRARY_PREFIX_A)/dccrg - -INC_EIGEN = -I$(LIBRARY_PREFIX_A)/eigen-eigen-ffa86ffb5570 - -LIB_PROFILE = -L$(LIBRARY_PREFIX_A)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_VERSION)/phiprof/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX_A)/phiprof/include - -INC_VECTORCLASS = -I$(LIBRARY_PREFIX_A)/vectorclass - -INC_VLSV = -I$(LIBRARY_PREFIX_A)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX_A)/vlsv -lvlsv - - diff --git a/MAKE/Makefile.arto b/MAKE/Makefile.arto deleted file mode 100644 index 18af9ec5a..000000000 --- a/MAKE/Makefile.arto +++ /dev/null @@ -1,36 +0,0 @@ -CMP = mpic++ -LNK = mpic++ - -LIBRARY_PREFIX = $(HOME)/libraries -FLAGS = -Wno-literal-suffix - -CXXFLAGS = -I$(LIBRARY_PREFIX)/include -O3 -funroll-loops -std=c++0x -fopenmp -fabi-version=0 -mavx -Wno-literal-suffix - -MATHFLAGS = -ffast-math -LDFLAGS = -L$(LIBRARY_PREFIX)/lib -LIB_MPI = -lgomp -PAPI_FLAG = - -INC_EIGEN = -I/usr/local/include/eigen3 - -INC_BOOST = -I/usr/local/include -LIB_BOOST = -L/usr/local/lib -lboost_program_options - - -INC_ZOLTAN = -I/usr/local/include -LIB_ZOLTAN = -lzoltan - -INC_VLSV = -I/home/sandroos/codes/vlsv -LIB_VLSV = -L/home/sandroos/codes/vlsv -lvlsv - -INC_SILO = -LIB_SILO = -lsilo - -INC_JEMALLOC = -LIB_JEMALLOC = -ljemalloc -INC_DCCRG = -I/home/sandroos/codes/dccrg - -LIB_PROFILE = -L/home/sandroos/codes/phiprof/lib -lphiprof -INC_PROFILE = -I/home/sandroos/codes/phiprof/include - -INC_VECTORCLASS = -I/home/sandroos/codes/vectorclass diff --git a/MAKE/Makefile.c2a_xlc b/MAKE/Makefile.c2a_xlc deleted file mode 100644 index 54ada80d9..000000000 --- a/MAKE/Makefile.c2a_xlc +++ /dev/null @@ -1,36 +0,0 @@ -CMP = mpCC -LNK = mpCC - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -LIB_MPI = - -BOOST_VERSION = 1.51.0 -ZOLTAN_VERSION = 3.6 -SILO_VERSION = 4.8 -XLC_VERSION = 12.1.0.0 - -LIBRARY_PREFIX = /perm/ms/fi/fmq/libraries - -INC_BOOST = -I$(LIBRARY_PREFIX)/xlc/$(XLC_VERSION)/boost/$(BOOST_VERSION)/include -LIB_BOOST = -L$(LIBRARY_PREFIX)/xlc/$(XLC_VERSION)/boost/$(BOOST_VERSION)/lib -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX)/xlc/$(XLC_VERSION)/zoltan/$(ZOLTAN_VERSION)/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/xlc/$(XLC_VERSION)/zoltan/$(ZOLTAN_VERSION)/lib -lzoltan - -INC_SILO = -I$(LIBRARY_PREFIX)/xlc/$(XLC_VERSION)/silo/$(SILO_VERSION)/include -LIB_SILO = -L$(LIBRARY_PREFIX)/xlc/$(XLC_VERSION)/silo/$(SILO_VERSION)/lib -lsilo - - -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg-2013-02-15 - - -LIB_PROFILE = -L$(LIBRARY_PREFIX)/xlc/$(XLC_VERSION)/phiprof/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX)/xlc/$(XLC_VERSION)/phiprof/include - -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass diff --git a/MAKE/Makefile.docker b/MAKE/Makefile.docker index 227109004..4eca04664 100644 --- a/MAKE/Makefile.docker +++ b/MAKE/Makefile.docker @@ -18,15 +18,27 @@ else VECTORCLASS = VEC4D_AGNER endif +FLAGS = + +#GNU flags: +CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -fabi-version=0 -mavx2 +testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 -mavx + +MATHFLAGS = -ffast-math +LDFLAGS = -g +LIB_MPI = -lgomp + #======== PAPI ========== #Add PAPI_MEM define to use papi to report memory consumption? CXXFLAGS += -DPAPI_MEM +testpackage: CXXFLAGS += -DPAPI_MEM #======== Allocator ========= #Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc #Configure jemalloc with --with-jemalloc-prefix=je_ when installing it CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE #======= Compiler and compilation flags ========= @@ -42,16 +54,7 @@ CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE # mpi.h in c++ on Cray CXXFLAGS += -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -#GNU flags: -CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++11 -W -Wall -Wno-unused -fabi-version=0 -mavx2 -testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++0x -fabi-version=0 -mavx - -MATHFLAGS = -ffast-math -LDFLAGS = -g -LIB_MPI = -lgomp +testpackage: CXXFLAGS += -DMPICH_IGNORE_CXX_SEEK # BOOST_VERSION = current trilinos version # ZOLTAN_VERSION = current trilinos verson diff --git a/MAKE/Makefile.fermi_gcc b/MAKE/Makefile.fermi_gcc deleted file mode 100644 index e30a4e3f0..000000000 --- a/MAKE/Makefile.fermi_gcc +++ /dev/null @@ -1,62 +0,0 @@ -CMP = mpicxx -LNK = mpicxx -# Use the two lines below to compile the not_parallel tools. -# CMP = g++ -# LNK = g++ - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = gcc -CC_BRAND_VERSION = 4.4.6 -CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++0x -W -Wall -pedantic -# CXXFLAGS += -O0 -W -Wall -pedantic -# CXXFLAGS += -O3 -funroll-loops -std=c++0x -W -Wall -pedantic -Wno-unused -# CXXFLAGS += -g -std=c++0x -W -Wall -pedantic -Wno-unused -# DCXXFLAGS += -g -fopenmp -std=c++0x -W -Wall -pedantic -Wno-unused -MATHFLAGS = -ffast-math -LDFLAGS = -LIB_MPI = -lgomp - -BOOST_VERSION = 1.50.0 -ZOLTAN_VERSION = 3.6 -SILO_VERSION = 4.8 -GCC_VERSION = 4.4.6 - -LIBRARY_PREFIX = /gpfs/scratch/userexternal/ykempf00/code/libs - -INC_BOOST = -I$(LIBRARY_PREFIX)/boost/$(BOOST_VERSION)/gcc/$(GCC_VERSION)/include -LIB_BOOST = -L$(LIBRARY_PREFIX)/boost/$(BOOST_VERSION)/gcc/$(GCC_VERSION)/lib -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/$(ZOLTAN_VERSION)/gcc/$(GCC_VERSION)/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/zoltan/$(ZOLTAN_VERSION)/gcc/$(GCC_VERSION)/lib -lzoltan - -INC_SILO = -I$(LIBRARY_PREFIX)/silo/$(SILO_VERSION)/gcc/$(GCC_VERSION)/include -LIB_SILO = -L$(LIBRARY_PREFIX)/silo/$(SILO_VERSION)/gcc/$(GCC_VERSION)/lib -lsilo - - -#Official dccrg installation -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg-2013-02-15 - - -LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/gcc/$(GCC_VERSION)/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/gcc/$(GCC_VERSION)/include - -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass - - - - diff --git a/MAKE/Makefile.fermi_xlc b/MAKE/Makefile.fermi_xlc deleted file mode 100644 index beff03381..000000000 --- a/MAKE/Makefile.fermi_xlc +++ /dev/null @@ -1,61 +0,0 @@ -CMP = mpixlcxx_r -LNK = mpixlcxx_r - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS = -qarch=qp -qtune=qp - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#XLC flags: -# CXXFLAGS += -O2 -pedantic -qsmp -qthreaded -qipa -qhot -qsimd -qunroll -qmaxmem=46829 # -qstrict -qlanglvl=extended -# CXXFLAGS += -O3 -pedantic -qsmp -qthreaded -qipa -qhot -qsimd -qunroll -# CXXFLAGS += -O3 -pedantic -qipa -qhot -qsimd -qunroll -CXXFLAGS += -O3 -qsmp=noauto:omp -qthreaded -qhot -qsimd -qunroll -# CXXFLAGS += -O3 -qhot -qsimd -qunroll -# CXXFLAGS += -O0 -# LDFLAGS = -qstaticlink -qipa -# LDFLAGS = -qstaticlink -qipa -qsmp -qthreaded -LDFLAGS = -qstaticlink -qsmp -qthreaded -# LDFLAGS = -qstaticlink -# LDFLAGS = -qstaticlink -LIB_MPI = - -BOOST_VERSION = 1.51.0 -ZOLTAN_VERSION = 3.6 -SILO_VERSION = 4.8 -XLC_VERSION = 1.0 - -LIBRARY_PREFIX = /gpfs/scratch/userexternal/ykempf00/code/libs - -INC_BOOST = -I$(LIBRARY_PREFIX)/boost/$(BOOST_VERSION)/xlc/$(XLC_VERSION)/include -LIB_BOOST = -L$(LIBRARY_PREFIX)/boost/$(BOOST_VERSION)/xlc/$(XLC_VERSION)/lib -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/$(ZOLTAN_VERSION)/xlc/$(XLC_VERSION)/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/zoltan/$(ZOLTAN_VERSION)/xlc/$(XLC_VERSION)/lib -lzoltan - -INC_SILO = -I$(LIBRARY_PREFIX)/silo/$(SILO_VERSION)/xlc/$(XLC_VERSION)/include -LIB_SILO = -L$(LIBRARY_PREFIX)/silo/$(SILO_VERSION)/xlc/$(XLC_VERSION)/lib -lsilo - - -#Official dccrg installation -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg-2013-02-15 - - -LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/xlc/$(XLC_VERSION)/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/xlc/$(XLC_VERSION)/include - -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass - - - - diff --git a/MAKE/Makefile.hawk_gcc_mpt b/MAKE/Makefile.hawk_gcc_mpt index 5f23d87c1..f7a35982a 100644 --- a/MAKE/Makefile.hawk_gcc_mpt +++ b/MAKE/Makefile.hawk_gcc_mpt @@ -1,19 +1,37 @@ CMP = mpicxx LNK = mpicxx +# module load gcc mpt papi boost +# Currently Loaded Modules: +# 1) system/site_names 2) system/ws/1.3.1 3) system/wrappers/1.0 4) gcc/9.2.0 5) mpt/2.23 6) papi/c048e224f 7) boost/1.70.0 +# +# in jobscript, use: +# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/zhome/academic/HLRS/pri/ipryakem/libraries/mpt/2.23/gcc/9.2.0/phiprof/lib +# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/zhome/academic/HLRS/pri/ipryakem/libraries/mpt/2.23/gcc/9.2.0/jemalloc/lib +# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/zhome/academic/HLRS/pri/ipryakem/libraries/mpt/2.23/gcc/9.2.0/zoltan/lib +# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/zhome/academic/HLRS/pri/ipryakem/libraries/mpt/2.23/gcc/9.2.0/vlsv/lib + + #======== Vectorization ========== #Set vector backend type for vlasov solvers, sets precision and length. #Options: # AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER # AVX512: VEC8D_AGNER, VEC16F_AGNER -# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK +# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK, VEC8D_FALLBACK ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) #Single-precision - VECTORCLASS = VEC8F_AGNER + VECTORCLASS = VEC8F_AGNER + INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass-version1 +# VECTORCLASS = VEC8F_FALLBACK +# INC_VECTORCLASS = -I$(HOME)/vlasiator/vlasovsolver else #Double-precision - VECTORCLASS = VEC8D_AGNER + VECTORCLASS = VEC4D_AGNER + INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass-version1 +# VECTORCLASS = VEC4D_FALLBACK +# VECTORCLASS = VEC8D_FALLBACK +# INC_VECTORCLASS = -I$(HOME)/vlasiator/vlasovsolver endif FLAGS = @@ -38,7 +56,7 @@ LIB_MPI = -lgomp BOOST_VERSION = MPI_BRAND = mpt -MPI_VERSION = 2.22 +MPI_VERSION = 2.23 ZOLTAN_VERSION = SILO_VERSION = JEMALLOC_VERSION = 5.2.1 @@ -48,18 +66,19 @@ LIBRARY_PREFIX = /zhome/academic/HLRS/pri/ipryakem/libraries #======== PAPI ========== #Add PAPI_MEM define to use papi to report memory consumption? CXXFLAGS += -DPAPI_MEM - +testpackage: CXXFLAGS += -DPAPI_MEM #======== Allocator ========= #Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc #Configure jemalloc with --with-jemalloc-prefix=je_ when installing it CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE #compiled libraries -INC_BOOST = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/include -LIB_BOOST = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/lib -lboost_program_options +LIB_BOOST= -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/lib -lboost_program_options +INC_BOOST=-L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/include INC_ZOLTAN = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/include LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/lib -lzoltan @@ -73,8 +92,8 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAN LIB_PROFILE = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib -lphiprof INC_PROFILE = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/include -INC_PAPI = -I/opt/hlrs/spack/rev-003_2020-03-03/papi/5.7.0-gcc-9.2.0-ie2ujpf4/include -LIB_PAPI = -L/opt/hlrs/spack/rev-003_2020-03-03/papi/5.7.0-gcc-9.2.0-ie2ujpf4/lib -lpapi +INC_PAPI = -I/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-gcc-9.2.0-hxfnx7kt/include +LIB_PAPI = -L/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-gcc-9.2.0-hxfnx7kt/lib -lpapi #header libraries @@ -83,6 +102,3 @@ INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/ INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid/ INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass-version1 - - - diff --git a/MAKE/Makefile.hawk_gcc_openmpi b/MAKE/Makefile.hawk_gcc_openmpi index bba350e6d..7f307a817 100644 --- a/MAKE/Makefile.hawk_gcc_openmpi +++ b/MAKE/Makefile.hawk_gcc_openmpi @@ -27,7 +27,6 @@ FLAGS = CC_BRAND = gcc CC_BRAND_VERSION = 9.2.0 CXXFLAGS += -g -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -fabi-version=0 -Wall -Wpedantic -mfma -march=native -mavx2 -#CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -fabi-version=0 -mavx not_parallel_tools: CXXFLAGS += -march=native -mno-avx2 -mavx testpackage: CXXFLAGS = -g -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 not_parallel_tools: CC_BRAND_VERSION = 4.9.2-noavx2 @@ -40,7 +39,7 @@ BOOST_VERSION = #MPI_BRAND = mpt #MPI_VERSION = 2.22 MPI_BRAND = openmpi -MPI_VERSION = 4.0.3 +MPI_VERSION = 4.0.4 ZOLTAN_VERSION = SILO_VERSION = JEMALLOC_VERSION = 5.2.1 @@ -50,23 +49,23 @@ LIBRARY_PREFIX = /zhome/academic/HLRS/pri/ipryakem/libraries #======== PAPI ========== #Add PAPI_MEM define to use papi to report memory consumption? CXXFLAGS += -DPAPI_MEM +testpackage: CXXFLAGS += -DPAPI_MEM #======== Allocator ========= #Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc #Configure jemalloc with --with-jemalloc-prefix=je_ when installing it CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE #compiled libraries -#INC_BOOST = -I/opt/hlrs/spack/rev-003_2020-03-03/boost/1.70.0-gcc-9.2.0-vf7vvxu2/include -#LIB_BOOST = -L/opt/hlrs/spack/rev-003_2020-03-03/boost/1.70.0-gcc-9.2.0-vf7vvxu2/lib -lboost_program_options -INC_BOOST = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/include -LIB_BOOST = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/lib -lboost_program_options +INC_BOOST = -I/opt/hlrs/spack/rev-004_2020-06-17/boost/1.70.0-gcc-9.2.0-ihlkdeae/include +LIB_BOOST = -L/opt/hlrs/spack/rev-004_2020-06-17/boost/1.70.0-gcc-9.2.0-ihlkdeae/lib -lboost_program_options -INC_ZOLTAN = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/lib -lzoltan +INC_ZOLTAN = -I/opt/hlrs/spack/rev-004_2020-06-17/trilinos/12.14.1-gcc-9.2.0-jp6yigfc/include +LIB_ZOLTAN = -L/opt/hlrs/spack/rev-004_2020-06-17/trilinos/12.14.1-gcc-9.2.0-jp6yigfc/lib -lzoltan INC_JEMALLOC = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/include LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/lib -ljemalloc @@ -77,8 +76,8 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAN LIB_PROFILE = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib -lphiprof INC_PROFILE = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/include -INC_PAPI = -I/opt/hlrs/spack/rev-003_2020-03-03/papi/5.7.0-gcc-9.2.0-ie2ujpf4/include -LIB_PAPI = -L/opt/hlrs/spack/rev-003_2020-03-03/papi/5.7.0-gcc-9.2.0-ie2ujpf4/lib -lpapi +INC_PAPI = -I/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-gcc-9.2.0-hxfnx7kt/include +LIB_PAPI = -L/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-gcc-9.2.0-hxfnx7kt/lib -lpapi #header libraries diff --git a/MAKE/Makefile.hawk_intel_mpt b/MAKE/Makefile.hawk_intel_mpt index f9ef3b9c3..189af98a3 100644 --- a/MAKE/Makefile.hawk_intel_mpt +++ b/MAKE/Makefile.hawk_intel_mpt @@ -26,6 +26,7 @@ FLAGS = #GNU flags: CC_BRAND = intel CC_BRAND_VERSION = 19.1.0 +# note: std was not updated to c++17 CXXFLAGS += -traceback -g -O3 -qopenmp -std=c++14 -W -Wall -Wno-unused -march=core-avx2 -qopt-zmm-usage=high testpackage: CXXFLAGS = -g -traceback -O2 -qopenmp -std=c++14 -W -Wno-unused -march=core-avx2 not_parallel_tools: CXXFLAGS += -march=native -mno-avx2 -mavx @@ -37,7 +38,7 @@ LIB_MPI = -lgomp BOOST_VERSION = MPI_BRAND = mpt -MPI_VERSION = 2.22 +MPI_VERSION = 2.23 ZOLTAN_VERSION = SILO_VERSION = JEMALLOC_VERSION = 5.2.1 @@ -47,12 +48,14 @@ LIBRARY_PREFIX = /zhome/academic/HLRS/pri/ipryakem/libraries #======== PAPI ========== #Add PAPI_MEM define to use papi to report memory consumption? CXXFLAGS += -DPAPI_MEM +testpackage: CXXFLAGS += -DPAPI_MEM #======== Allocator ========= #Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc #Configure jemalloc with --with-jemalloc-prefix=je_ when installing it CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE #compiled libraries @@ -74,8 +77,8 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAN LIB_PROFILE = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib -lphiprof INC_PROFILE = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/include -INC_PAPI = -I/opt/hlrs/spack/rev-003_2020-03-03/papi/5.7.0-intel-19.1.0-afazikxh/include -LIB_PAPI = -L/opt/hlrs/spack/rev-003_2020-03-03/papi/5.7.0-intel-19.1.0-afazikxh/lib -lpapi +INC_PAPI = -I/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-intel-19.1.0-wlgbb2mr/include +LIB_PAPI = -L/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-intel-19.1.0-wlgbb2mr/lib -lpapi #header libraries diff --git a/MAKE/Makefile.hawk_intel_openmpi b/MAKE/Makefile.hawk_intel_openmpi index fa7361cca..3a5b4b25a 100644 --- a/MAKE/Makefile.hawk_intel_openmpi +++ b/MAKE/Makefile.hawk_intel_openmpi @@ -26,6 +26,7 @@ FLAGS = #GNU flags: CC_BRAND = intel CC_BRAND_VERSION = 19.1.0 +#note: std was not updated to c++17 CXXFLAGS += -traceback -g -O3 -qopenmp -std=c++14 -W -Wall -Wno-unused -march=core-avx2 -qopt-zmm-usage=high testpackage: CXXFLAGS = -g -traceback -O2 -qopenmp -std=c++14 -W -Wno-unused -march=core-avx2 not_parallel_tools: CXXFLAGS += -march=native -mno-avx2 -mavx @@ -39,7 +40,7 @@ BOOST_VERSION = #MPI_BRAND = mpt #MPI_VERSION = 2.22 MPI_BRAND = openmpi -MPI_VERSION = 4.0.3 +MPI_VERSION = 4.0.4 ZOLTAN_VERSION = SILO_VERSION = JEMALLOC_VERSION = 5.2.1 @@ -49,12 +50,14 @@ LIBRARY_PREFIX = /zhome/academic/HLRS/pri/ipryakem/libraries #======== PAPI ========== #Add PAPI_MEM define to use papi to report memory consumption? CXXFLAGS += -DPAPI_MEM +testpackage: CXXFLAGS += -DPAPI_MEM #======== Allocator ========= #Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc #Configure jemalloc with --with-jemalloc-prefix=je_ when installing it CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE #compiled libraries @@ -74,8 +77,8 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAN LIB_PROFILE = -L$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib -lphiprof INC_PROFILE = -I$(LIBRARY_PREFIX)/$(MPI_BRAND)/$(MPI_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/include -INC_PAPI = -I/opt/hlrs/spack/rev-003_2020-03-03/papi/5.7.0-intel-19.1.0-afazikxh/include -LIB_PAPI = -L/opt/hlrs/spack/rev-003_2020-03-03/papi/5.7.0-intel-19.1.0-afazikxh/lib -lpapi +INC_PAPI = -I/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-intel-19.1.0-wlgbb2mr/include +LIB_PAPI = -L/opt/hlrs/spack/rev-004_2020-06-17/papi/c048e224f-intel-19.1.0-wlgbb2mr/lib -lpapi #header libraries diff --git a/MAKE/Makefile.hermit_cray b/MAKE/Makefile.hermit_cray deleted file mode 100644 index 2c61e0b90..000000000 --- a/MAKE/Makefile.hermit_cray +++ /dev/null @@ -1,58 +0,0 @@ -CMP = CC -LNK = CC - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = cray -CC_BRAND_VERSION = 8.1.1 -CXXFLAGS += -O3 -MATHFLAGS = -LDFLAGS = -LIB_MPI = - -BOOST_VERSION = 1.52.0 -MPT_VERSION = 5.5.5 -ZOLTAN_VERSION = 3.6 -SILO_VERSION = 4.8 - -LIBRARY_PREFIX_A = /zhome/academic/HLRS/pri/iprilhon/libraries -LIBRARY_PREFIX_B = /zhome/academic/HLRS/pri/iprsalft/libraries - - - -INC_BOOST = -I$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/include -LIB_BOOST = -L$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/lib -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/lib -lzoltan - -INC_SILO = -I$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/include -LIB_SILO = -L$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/lib -lsilo - - -#Official dccrg installation -INC_DCCRG = -I$(LIBRARY_PREFIX_A)/dccrg-2013-02-20-cray-workarounds - - -LIB_PROFILE = -L$(LIBRARY_PREFIX_B)/phiprof/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX_B)/phiprof/include - -INC_VECTORCLASS = -I$(LIBRARY_PREFIX_B)/vectorclass - - - - diff --git a/MAKE/Makefile.hermit_gcc b/MAKE/Makefile.hermit_gcc deleted file mode 100644 index 7de88fc5d..000000000 --- a/MAKE/Makefile.hermit_gcc +++ /dev/null @@ -1,61 +0,0 @@ -CMP = CC -LNK = CC - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = gcc -CC_BRAND_VERSION = 4.7.2 -CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++0x -W -Wall -pedantic -Wno-unused -fabi-version=6 -mfma4 -DEIGEN_NO_DEBUG -# CXXFLAGS += -g -O0 -funroll-loops -fopenmp -std=c++0x -fabi-version=6 -mfma4 -MATHFLAGS = -ffast-math -LDFLAGS = -LIB_MPI = -lgomp - -BOOST_VERSION = 1.53.0 -MPT_VERSION = 5.6.1 -ZOLTAN_VERSION = 3.6 -SILO_VERSION = 4.8 - -LIBRARY_PREFIX_A = /zhome/academic/HLRS/pri/iprilhon/libraries -LIBRARY_PREFIX_B = /zhome/academic/HLRS/pri/iprsalft/libraries - -INC_VLSV = -I$(LIBRARY_PREFIX_B)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX_B)/vlsv -lvlsv - -INC_BOOST = -I$(LIBRARY_PREFIX_A)/cray-mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/include -LIB_BOOST = -L$(LIBRARY_PREFIX_A)/cray-mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/lib -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX_A)/cray-mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX_A)/cray-mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/lib -lzoltan - -INC_SILO = -I$(LIBRARY_PREFIX_A)/cray-mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/include -LIB_SILO = -L$(LIBRARY_PREFIX_A)/cray-mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/lib -lsilo - - -#Official dccrg installation -INC_DCCRG = -I$(LIBRARY_PREFIX_A)/dccrg-2013-02-20-cray-workarounds - -INC_EIGEN = -I$(LIBRARY_PREFIX_B)/eigen/eigen-eigen-2249f9c22fe8 - -LIB_PROFILE = -L$(LIBRARY_PREFIX_B)/cray-mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX_B)/cray-mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/include - -INC_VECTORCLASS = -I$(LIBRARY_PREFIX_B)/vectorclass - - - - diff --git a/MAKE/Makefile.hermit_intel b/MAKE/Makefile.hermit_intel deleted file mode 100644 index 6b469cdf2..000000000 --- a/MAKE/Makefile.hermit_intel +++ /dev/null @@ -1,58 +0,0 @@ -CMP = CC -LNK = CC - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = intel -CC_BRAND_VERSION = 13.0.0 -CXXFLAGS += -O3 -openmp -MATHFLAGS = -LDFLAGS = -openmp -LIB_MPI = - -BOOST_VERSION = 1.52.0 -MPT_VERSION = 5.5.5 -ZOLTAN_VERSION = 3.6 -SILO_VERSION = 4.8 - -LIBRARY_PREFIX_A = /zhome/academic/HLRS/pri/iprilhon/libraries -LIBRARY_PREFIX_B = /zhome/academic/HLRS/pri/iprsalft/libraries - - - -INC_BOOST = -I$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/include -LIB_BOOST = -L$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/lib -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/lib -lzoltan - -INC_SILO = -I$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/include -LIB_SILO = -L$(LIBRARY_PREFIX_A)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/lib -lsilo - - -#Official dccrg installation -INC_DCCRG = -I$(LIBRARY_PREFIX_A)/dccrg-2013-02-20-cray-workarounds - - -LIB_PROFILE = -L$(LIBRARY_PREFIX_B)/phiprof/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX_B)/phiprof/include - -INC_VECTORCLASS = -I$(LIBRARY_PREFIX_B)/vectorclass - - - - diff --git a/MAKE/Makefile.home b/MAKE/Makefile.home index 8934fd3af..1f50eef47 100644 --- a/MAKE/Makefile.home +++ b/MAKE/Makefile.home @@ -2,15 +2,17 @@ CMP = mpic++ LNK = mpic++ FLAGS = -CXXFLAGS = -I$(HOME)/include -I/usr/include -L$(HOME)/lib -L/usr/lib -O3 -funroll-loops -std=c++11 -W -Wall -pedantic -Wno-unused -Wno-unused-parameter -Wno-missing-braces -fopenmp -#CXXFLAGS = -I$(HOME)/include -L$(HOME)/lib -O0 -funroll-loops -std=c++11 -W -Wall -pedantic -Wno-unused -Wno-unused-parameter -Wno-missing-braces -g -fopenmp +# note: std was c++11 +# note: testpackage settings missing +CXXFLAGS = -I$(HOME)/include -I/usr/include -L$(HOME)/lib -L/usr/lib -O3 -funroll-loops -std=c++17 -W -Wall -pedantic -Wno-unused -Wno-unused-parameter -Wno-missing-braces -fopenmp +#CXXFLAGS = -I$(HOME)/include -L$(HOME)/lib -O0 -funroll-loops -std=c++17 -W -Wall -pedantic -Wno-unused -Wno-unused-parameter -Wno-missing-braces -g -fopenmp MATHFLAGS = -ffast-math LDFLAGS = -L $(HOME)/lib LIB_MPI = -lgomp # LIB_MPI = -LIBRARY_PREFIX = $HOME -LIBRARY_PREFIX_B = $HOME +LIBRARY_PREFIX = $(HOME) +LIBRARY_PREFIX_B = $(HOME) INC_BOOST = -I${HOME}/include LIB_BOOST = -L${HOME}/lib -lboost_program_options diff --git a/MAKE/Makefile.hornet_gcc b/MAKE/Makefile.hornet_gcc deleted file mode 100644 index 4a4418332..000000000 --- a/MAKE/Makefile.hornet_gcc +++ /dev/null @@ -1,89 +0,0 @@ -CMP = CC -LNK = CC - -#======== Vectorization ========== -#Set vector backend type for vlasov solvers, sets precision and length. -#Options: -# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER -# AVX512: VEC8D_AGNER, VEC16F_AGNER -# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK - -ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) -#Single-precision - VECTORCLASS = VEC16F_AGNER -else -#Double-precision - VECTORCLASS = VEC8D_AGNER -endif - -#======== PAPI ========== -#Add PAPI_MEM define to use papi to report memory consumption? -CXXFLAGS += -DPAPI_MEM - - -#======== Allocator ========= -#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc -#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it -CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE - - -CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = gcc -CC_BRAND_VERSION = 8.3.0 -CXXFLAGS += -g -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -fabi-version=0 -mavx2 -Wall -Wpedantic -#CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -fabi-version=0 -mavx -not_parallel_tools: CXXFLAGS += -march=native -mno-avx2 -mavx -testpackage: CXXFLAGS = -g -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 -not_parallel_tools: CC_BRAND_VERSION = 4.9.2-noavx2 - -MATHFLAGS = -ffast-math -LDFLAGS = -LIB_MPI = -lgomp - -BOOST_VERSION = -MPT_VERSION = 7.7.6 -ZOLTAN_VERSION = -SILO_VERSION = -JEMALLOC_VERSION = 5.2.1 -LIBRARY_PREFIX = /zhome/academic/HLRS/pri/ipryakem/libraries - - -#compiled libraries - -INC_BOOST = -I$(CRAY_TRILINOS_PREFIX_DIR)/include/boost -LIB_BOOST = -L$(CRAY_TRILINOS_PREFIX_DIR)/lib -lboost_program_options - -INC_ZOLTAN = -I$(CRAY_TRILINOS_PREFIX_DIR)/include -LIB_ZOLTAN = -L$(CRAY_TRILINOS_PREFIX_DIR)/lib -lzoltan - -#INC_SILO = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/include -#LIB_SILO = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/lib -lsilo - -INC_JEMALLOC = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/include -LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/lib -ljemalloc - -INC_VLSV = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -lvlsv - -LIB_PROFILE = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/include - -#header libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX)/ -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/ -INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid/ -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass-version1 - - - - diff --git a/MAKE/Makefile.hornet_intel b/MAKE/Makefile.hornet_intel deleted file mode 100644 index a2ae7ce65..000000000 --- a/MAKE/Makefile.hornet_intel +++ /dev/null @@ -1,83 +0,0 @@ -#Intel compile on hornet, Cray XC40 -#List of required module loads & swaps -#module swap PrgEnv-cray PrgEnv-intel -#module swap intel intel/15.0.2.164 -#module load cray-trilinos -#module load papi -#module load gcc/4.9.2 - - - -CMP = CC -LNK = CC - -not_parallel_tools: CMP = icpc -not_parallel_tools: LNK = icpc - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = intel -CC_BRAND_VERSION = 15.0.2 -#With intel 15 these options are a few % faster than gnu 4.9 -CXXFLAGS += -Ofast -inline-level=2 -no-inline-min-size -std=c++11 -openmp -ansi-alias -#CXXFLAGS += -O2 -std=c++11 -openmp -ansi-alias - - - - -MATHFLAGS = -LDFLAGS = -openmp -LIB_MPI = - -BOOST_VERSION = 1.56.0 -MPT_VERSION = 7.1.3 -ZOLTAN_VERSION = 3.8 -SILO_VERSION = 4.9.1 -JEMALLOC_VERSION = 3.6.0 -LIBRARY_PREFIX = /zhome/academic/HLRS/pri/iprurgan/vlasiator/libraries - - -#compiled libraries -INC_BOOST = -I$(CRAY_TRILINOS_PREFIX_DIR)/include/boost -INC_BOOST = -L$(CRAY_TRILINOS_PREFIX_DIR)/lib -lboost_program_options - -INC_ZOLTAN = -I$(CRAY_TRILINOS_PREFIX_DIR)/include -LIB_ZOLTAN = -I$(CRAY_TRILINOS_PREFIX_DIR)/lib -lzoltan - - -INC_SILO = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/include -LIB_SILO = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/lib -lsilo - -INC_JEMALLOC = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/include -LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/lib -ljemalloc - -INC_VLSV = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -lvlsv - - -LIB_PROFILE = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/include - -#header libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen/ -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/ -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass - - - - diff --git a/MAKE/Makefile.jugene b/MAKE/Makefile.jugene deleted file mode 100644 index 7d9c5830b..000000000 --- a/MAKE/Makefile.jugene +++ /dev/null @@ -1,48 +0,0 @@ - -CMP = mpicxx -LNK = mpicxx - -CXXFLAGS = #-DMPICH_IGNORE_CXX_SEEK -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for mandatory compiler flags, they are always used -# FLAGS is for user-defined flags, this must be left empty on all Makefiles -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#XLC flags: -#CC_BRAND = xlc -#CC_BRAND_VERSION = 9.0 -#CXXFLAGS += -O2 -qarch=450 -qtune=450 -#MATHFLAGS = -#LDFLAGS = -#LIB_MPI = - -#GNU flags -CC_BRAND = gcc -CC_BRAND_VERSION = 4.1.2 -CXXFLAGS += -O3 -funroll-loops -MATHFLAGS = -ffast-math -LDFLAGS = -LIB_MPI = - -#CXXFLAGS += -DTOPOLOGY_OPTIMIZATION -DIBM_BGP - -BOOST_VERSION = 1.47.0 -ZOLTAN_VERSION = 3.501 -SILO_VERSION = 4.8 - -LIBRARY_PREFIX = /homeb/prpc03/prpc0301/libraries/ - -INC_BOOST = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/include -LIB_BOOST = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/lib -lboost_mpi -lboost_serialization -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/lib -lzoltan - -INC_SILO = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/include -LIB_SILO = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/lib -lsilo - -INC_DCCRG = -I/homeb/prpc03/prpc0301/dccrg-2013-02-15 -INC_TOPO = -I/homeb/prpc03/prpc0300/QuESpace/alfthan/topologyOptimization_b diff --git a/MAKE/Makefile.kstppd b/MAKE/Makefile.kstppd new file mode 100644 index 000000000..997bc5ab2 --- /dev/null +++ b/MAKE/Makefile.kstppd @@ -0,0 +1,90 @@ +#Kostis Laptop +#Compiler & Linker +CMP = mpic++ +LNK = mpic++ + + +#======== Vectorization ========== +#Set vector backend type for vlasov solvers, sets precision and length. +#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) +#Options; +# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER +# AVX512: VEC8D_AGNER, VEC16F_AGNER +# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK + +ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) +#Single-precision + VECTORCLASS = VEC8F_AGNER +else +#Double-precision + VECTORCLASS = VEC4D_AGNER +endif + + + +#======== Flags ========= + +FLAGS = +CXXFLAGS = -O3 -funroll-loops -std=c++17 -fopenmp -W -Wall -Wno-unused -Wno-unused-parameter -Wno-missing-braces -fabi-version=0 -mavx +testpackage: CXXFLAGS = -g -ggdb -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 -mno-avx -mno-fma -fno-unsafe-math-optimizations +CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE -DPAPI_MEM +MATHFLAGS = -ffast-math +testpackage: MATHFLAGS = -fno-unsafe-math-optimizations +LDFLAGS = -L $(HOME)/lib +LIB_MPI = -lgomp -lpapi + + + + +#================PAPI================== +#Add PAPI_MEM define to use papi to report memory consumption? +CXXFLAGS += -DPAPI_MEM +testpackage: CXXFLAGS += -DPAPI_MEM + + +#======== Allocator ========= +#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc +#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it +CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE + +#======== Allocator ========= +#Use TBB malloc + + +PAPI_FLAG = + + +#======== Libraries =========== + +LIBRARY_PREFIX = $(HOME) + +INC_VECTORCLASS=-I$(LIBRARY_PREFIX)/version1 + +INC_BOOST = -I/usr/include/boost/ +LIB_BOOST = -L/usr/lib/x86_64-linux-gnu/ -lboost_program_options + +INC_ZOLTAN = -I$(LIBRARY_PREFIX)/include +LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/lib -lzoltan + +INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv +LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv + +INC_SILO = -I$(LIBRARY_PREFIX)/include +LIB_SILO = -L$(LIBRARY_PREFIX)/lib -lsilo + +INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg + +INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid + +INC_PAPI = -I$(LIBRARY_PREFIX)/papi-5.4.3/src +LIB_PAPI = -L$(LIBRARY_PREFIX)/papi-5.4.3/src -lpapi + +INC_JEMALLOC = -I$(LIBRARY_PREFIX)/libraries/jemalloc +LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/libraries/jemalloc/lib -ljemalloc + +LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof +INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include + +INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen-eigen-07105f7124f9 + diff --git a/MAKE/Makefile.louhi_intel b/MAKE/Makefile.louhi_intel deleted file mode 100644 index 317c92430..000000000 --- a/MAKE/Makefile.louhi_intel +++ /dev/null @@ -1,33 +0,0 @@ -MOVER=cpu - -CMP = CC -LNK = CC -AR = xiar - -LIB_MPI = - -FLAGS = -DMPICH_IGNORE_CXX_SEEK - -CC_BRAND = intel -CC_BRAND_VERSION = 12.0.3.174 -FLAGS += -O3 -fast -openmp -DNDEBUG -LDFLAGS = -static -openmp - -BOOST_VERSION = 1.46.1 -MPT_VERSION = 5.2.1 -ZOLTAN_VERSION = 3.5 -SILO_VERSION = 4.8 - -ROOT_DIR=/v/users/ilhonkon/libraries/ - -INC_BOOST = -I$(ROOT_DIR)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/include -LIB_BOOST = -L$(ROOT_DIR)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/lib -lboost_mpi -lboost_serialization -lboost_program_options - -INC_ZOLTAN = -I$(ROOT_DIR)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/include -LIB_ZOLTAN = -L$(ROOT_DIR)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/lib -lzoltan - -INC_SILO = -I$(ROOT_DIR)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/include -LIB_SILO = -L$(ROOT_DIR)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/lib -lsilo - -INC_DCCRG = -I$(ROOT_DIR)/dccrg-2013-02-15 - diff --git a/MAKE/Makefile.appa b/MAKE/Makefile.mahti_gcc similarity index 51% rename from MAKE/Makefile.appa rename to MAKE/Makefile.mahti_gcc index 670daf45c..63764900c 100644 --- a/MAKE/Makefile.appa +++ b/MAKE/Makefile.mahti_gcc @@ -1,10 +1,11 @@ -# -*- mode: makefile -*- CMP = mpic++ LNK = mpic++ +# Modules loaded +# module load gcc boost jemalloc papi openmpi zoltan + #======== Vectorization ========== #Set vector backend type for vlasov solvers, sets precision and length. -#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) #Options: # AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER # AVX512: VEC8D_AGNER, VEC16F_AGNER @@ -18,16 +19,28 @@ else VECTORCLASS = VEC4D_AGNER endif +FLAGS = + +#GNU flags: +CC_BRAND = gcc +CC_BRAND_VERSION = 9.3.0 +CXXFLAGS += -g -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -fabi-version=0 -march=native -mfma -mavx2 +testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 -mfma + +MATHFLAGS = -ffast-math +LDFLAGS = -lrt +LIB_MPI = -lgomp + #======== PAPI ========== #Add PAPI_MEM define to use papi to report memory consumption? CXXFLAGS += -DPAPI_MEM - +testpackage: CXXFLAGS += -DPAPI_MEM #======== Allocator ========= #Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc #Configure jemalloc with --with-jemalloc-prefix=je_ when installing it CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE - +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE #======= Compiler and compilation flags ========= # NOTES on compiler flags: @@ -37,61 +50,36 @@ CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE #-DNO_WRITE_AT_ALL: Define to disable write at all to # avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS += -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -#GNU flags: -CC_BRAND = gcc -CC_BRAND_VERSION = 5.4.0 -CXXFLAGS += -g -O2 -fopenmp -funroll-loops -std=c++11 -W -Wall -Wno-unused -fabi-version=0 -mavx2 -#CXXFLAGS += -g -fopenmp -funroll-loops -std=c++11 -W -Wall -Wno-unused -fabi-version=0 -mavx2 -testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++0x -fabi-version=0 -mavx - -MATHFLAGS = -ffast-math -LDFLAGS = -g -LIB_MPI = -lgomp # BOOST_VERSION = current trilinos version # ZOLTAN_VERSION = current trilinos verson - +# #======== Libraries =========== -MPT_VERSION = 7.2.6 -JEMALLOC_VERSION = 5.0.1 -LIBRARY_PREFIX = /home/tkoskela/lib +MPT_VERSION = 4.0.3 +JEMALLOC_VERSION = 5.2.1 +LIBRARY_PREFIX = /users/kempf/libraries #compiled libraries -INC_BOOST = -I$/usr/include/boost -LIB_BOOST = -L$/usr/lib/x86_64-linux-gnu -lboost_program_options +LIB_BOOST = -lboost_program_options + +LIB_ZOLTAN = -lzoltan -INC_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/Zoltan_v3.83/build/include -#LIB_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/Zoltan_v3.83/build/lib -lzoltan -LIB_ZOLTAN = /home/tkoskela/lib/zoltan/Zoltan_v3.83/build/lib/libzoltan.a +LIB_JEMALLOC = -ljemalloc -INC_JEMALLOC = -I$(LIBRARY_PREFIX)/jemalloc/include -LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/jemalloc/lib -ljemalloc +LIB_PAPI = -lpapi INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv -INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/phiprof-2.0-beta/include -LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/phiprof-2.0-beta/lib -lphiprof +LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -lgfortran -Wl,-rpath=$(LIBRARY_PREFIX)/phiprof/lib +INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include -INC_PAPI = -I$(LIBRARY_PREFIX)/papi/include -LIB_PAPI = -L$(LIBRARY_PREFIX)/papi/lib -lpapi #header libraries +INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid/ INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen/ INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/ -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass -INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid - - - +INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass/ diff --git a/MAKE/Makefile.marconi b/MAKE/Makefile.marconi index 81a9b2dce..6cc390635 100644 --- a/MAKE/Makefile.marconi +++ b/MAKE/Makefile.marconi @@ -25,15 +25,28 @@ else VECTORCLASS = VEC8D_AGNER endif +#GNU flags: +CC_BRAND = intel +CC_BRAND_VERSION = 17.0.1 +# note: std was c++11 +CXXFLAGS += -O2 -g -DMAX_VECTOR_SIZE=512 -xMIC-AVX512 -std=c++17 -qopenmp -ansi-alias +testpackage: CXXFLAGS = -O2 -g -DMAX_VECTOR_SIZE=512 -xMIC-AVX512 -std=c++17 -qopenmp -ansi-alias +MATHFLAGS = +LDFLAGS += -qopenmp -lifcore -Wl,-rpath,/cineca/prod/opt/libraries/boost/1.61.0/intelmpi--2017--binary/lib,-rpath,/marconi_work/Pra14_3521/libraries/phiprof/lib,-rpath,/marconi_work/Pra14_3521/libraries/papi/lib +CMP = mpicxx +LNK = mpicxx + #======== PAPI ========== #Add PAPI_MEM define to use papi to report memory consumption? CXXFLAGS += -DPAPI_MEM +testpackage: CXXFLAGS += -DPAPI_MEM #======== Allocator ========= #Use TBB malloc #LDFLAGS += -L$(TBBROOT)/lib/intel64/gcc4.7/ -ltbbmalloc_proxy -ltbbmalloc CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE #======= Compiler and compilation flags ========= @@ -41,16 +54,6 @@ CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE # CXXFLAGS is for compiler flags, they are always used # MATHFLAGS are for special math etc. flags, these are only applied on solver functions # LDFLAGS flags for linker -#GNU flags: -CC_BRAND = intel -CC_BRAND_VERSION = 17.0.1 -CXXFLAGS += -O2 -g -DMAX_VECTOR_SIZE=512 -xMIC-AVX512 -std=c++11 -qopenmp -ansi-alias -testpackage: CXXFLAGS = -O2 -g -DMAX_VECTOR_SIZE=512 -xMIC-AVX512 -std=c++11 -qopenmp -ansi-alias -MATHFLAGS = -LDFLAGS += -qopenmp -lifcore -Wl,-rpath,/cineca/prod/opt/libraries/boost/1.61.0/intelmpi--2017--binary/lib,-rpath,/marconi_work/Pra14_3521/libraries/phiprof/lib,-rpath,/marconi_work/Pra14_3521/libraries/papi/lib -CMP = mpicxx -LNK = mpicxx - #======== Libraries =========== diff --git a/MAKE/Makefile.meteo b/MAKE/Makefile.meteo deleted file mode 100644 index 03f9151fe..000000000 --- a/MAKE/Makefile.meteo +++ /dev/null @@ -1,72 +0,0 @@ -CMP = CC -LNK = CC - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK -DNDEBUG -DEIGEN_NO_DEBUG - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = gcc -CC_BRAND_VERSION = 4.5.3 -CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++0x -W -Wall -pedantic -Wno-unused -# CXXFLAGS += -O3 -funroll-loops -std=c++0x -W -Wall -pedantic -Wno-unused -#CXXFLAGS += -g -fopenmp -std=c++0x -W -Wall -pedantic -Wno-unused -MATHFLAGS = -ffast-math -LDFLAGS = -LIB_MPI = -lgomp - -BOOST_VERSION = 1.50.0 -MPT_VERSION = 5.3.5 -ZOLTAN_VERSION = 3.6 -SILO_VERSION = 4.8 - -LIBRARY_PREFIX = /stornext/field/users/honkonen/libraries -LIBRARY_PREFIX_B = /stornext/field/users/alfthan/libraries -LIBRARY_PREFIX_C = /stornext/field/users/hannukse/libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX_B)/eigen/eigen-eigen-2249f9c22fe8 - -INC_VLSV = -I$(LIBRARY_PREFIX_C)/vlsv/trunk -LIB_VLSV = -L$(LIBRARY_PREFIX_C)/vlsv/trunk -lvlsv - -INC_BOOST = -I$(LIBRARY_PREFIX)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/include -LIB_BOOST = -L$(LIBRARY_PREFIX)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/$(BOOST_VERSION)/lib -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/$(ZOLTAN_VERSION)/lib -lzoltan - -INC_SILO = -I$(LIBRARY_PREFIX)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/include -LIB_SILO = -L$(LIBRARY_PREFIX)/mpt/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/lib -lsilo - - -INC_EIGEN = -I$(LIBRARY_PREFIX_B)/eigen/eigen-eigen-2249f9c22fe8 - - -#DCCRG version with reduced neighbourhood. Might not be up-to-date -#INC_DCCRG = -I$(LIBRARY_PREFIX_B)/dccrg-new - -#Official dccrg installation -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg-2013-02-20-cray-workarounds -# INC_DCCRG = -Idccrg - - -LIB_PROFILE = -L$(LIBRARY_PREFIX_B)/phiprof/lib -lphiprof - -INC_PROFILE = -I$(LIBRARY_PREFIX_B)/phiprof/include - -INC_VECTORCLASS = -I$(LIBRARY_PREFIX_B)/vectorclass - - - - diff --git a/MAKE/Makefile.ninja b/MAKE/Makefile.ninja deleted file mode 100644 index b870c0e58..000000000 --- a/MAKE/Makefile.ninja +++ /dev/null @@ -1,60 +0,0 @@ -CMP = mpicxx -LNK = mpicxx - - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = intel -CC_BRAND_VERSION = 17-openmpi-2.0.1 -#CXXFLAGS += -O2 -xAVX2 -std=c++11 -qopenmp -ansi-alias -CXXFLAGS += -O2 -g -DMAX_VECTOR_SIZE=512 -xMIC-AVX512 -std=c++11 -qopenmp -ansi-alias -testpackage: CXXFLAGS = -O2 -std=c++11 -qopenmp -ansi-alias - - -MATHFLAGS = -LDFLAGS = -qopenmp -lifcore -L$(TBBROOT)/lib/intel64/gcc4.7/ -ltbbmalloc_proxy -ltbbmalloc - - -HEADER_LIBRARY_PREFIX = /scratch/alfthan/opt/ -LIBRARY_PREFIX = /scratch/alfthan/opt/intel-$(CC_BRAND_VERSION) - - -#compiled libraries - -INC_BOOST = -I$(LIBRARY_PREFIX)/boost/include -LIB_BOOST = -L$(LIBRARY_PREFIX)/boost/lib -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/zoltan/lib -lzoltan - - -INC_JEMALLOC = -I$(LIBRARY_PREFIX)/jemalloc/include -LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/jemalloc/lib -ljemalloc - -INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv - - - -LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/2.0/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/2.0/include - -#system -LIB_PAPI = -L/usr/lib64/ -lpapi - -#header libraries - -INC_EIGEN = -I$(HEADER_LIBRARY_PREFIX)/eigen-eigen-dc6cfdf9bcec -INC_DCCRG = -I$(HEADER_LIBRARY_PREFIX)/dccrg/ -INC_VECTORCLASS = -I$(HEADER_LIBRARY_PREFIX)/vectorclass - - - - - diff --git a/MAKE/Makefile.puhti_gcc b/MAKE/Makefile.puhti_gcc index 406664b70..002bb1367 100644 --- a/MAKE/Makefile.puhti_gcc +++ b/MAKE/Makefile.puhti_gcc @@ -16,16 +16,6 @@ else VECTORCLASS = VEC8D_AGNER endif - -#======= Compiler and compilation flags ========= -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) - FLAGS = #GNU flags: @@ -38,25 +28,36 @@ MATHFLAGS = -ffast-math LDFLAGS = -lrt LIB_MPI = -lgomp -# BOOST_VERSION = current trilinos version -# ZOLTAN_VERSION = current trilinos verson -# -#======== Libraries =========== - -MPT_VERSION = 2.4.0 -JEMALLOC_VERSION = 5.2.1 -LIBRARY_PREFIX = /projappl/project_2000203/libraries +#======= Compiler and compilation flags ========= +# NOTES on compiler flags: +# CXXFLAGS is for compiler flags, they are always used +# MATHFLAGS are for special math etc. flags, these are only applied on solver functions +# LDFLAGS flags for linker +# +#-DNO_WRITE_AT_ALL: Define to disable write at all to +# avoid memleak (much slower IO) #======== PAPI ========== #Add PAPI_MEM define to use papi to report memory consumption? #CXXFLAGS += -DPAPI_MEM - +#testpackage: CXXFLAGS += -DPAPI_MEM #======== Allocator ========= #Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc #Configure jemalloc with --with-jemalloc-prefix=je_ when installing it CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE + + +# BOOST_VERSION = current trilinos version +# ZOLTAN_VERSION = current trilinos verson +# +#======== Libraries =========== + +MPT_VERSION = 2.4.0 +JEMALLOC_VERSION = 5.2.1 +LIBRARY_PREFIX = /projappl/project_2000203/libraries #compiled libraries diff --git a/MAKE/Makefile.puhti_intel b/MAKE/Makefile.puhti_intel index 3db410831..031a800f3 100644 --- a/MAKE/Makefile.puhti_intel +++ b/MAKE/Makefile.puhti_intel @@ -16,16 +16,28 @@ else VECTORCLASS = VEC8D_AGNER endif +FLAGS = + +#GNU flags: +CC_BRAND = intel +CC_BRAND_VERSION = 19.0.4 +CXXFLAGS += -traceback -g -O3 -qopenmp -std=c++17 -W -Wall -Wno-unused -xHost -ipo -qopt-zmm-usage=high +testpackage: CXXFLAGS = -g -traceback -O2 -qopenmp -std=c++17 -W -Wno-unused -xHost -ipo + +MATHFLAGS = -ffast-math +LDFLAGS = -qopenmp -lifcore -ipo +LIB_MPI = + #======== PAPI ========== #Add PAPI_MEM define to use papi to report memory consumption? #CXXFLAGS += -DPAPI_MEM - +#testpackage: CXXFLAGS += -DPAPI_MEM #======== Allocator ========= #Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc #Configure jemalloc with --with-jemalloc-prefix=je_ when installing it CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE - +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE #======= Compiler and compilation flags ========= # NOTES on compiler flags: @@ -40,18 +52,7 @@ CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE # mpi.h in c++ on Cray CXXFLAGS += -DMAX_VECTOR_SIZE=512 - -FLAGS = - -#GNU flags: -CC_BRAND = intel -CC_BRAND_VERSION = 19.0.4 -CXXFLAGS += -traceback -g -O3 -qopenmp -std=c++17 -W -Wall -Wno-unused -xHost -ipo -qopt-zmm-usage=high -testpackage: CXXFLAGS += -g -traceback -O2 -qopenmp -std=c++17 -W -Wno-unused -xHost -ipo - -MATHFLAGS = -ffast-math -LDFLAGS = -qopenmp -lifcore -ipo -LIB_MPI = +testpackage: CXXFLAGS += -DMAX_VECTOR_SIZE=512 # BOOST_VERSION = current trilinos version # ZOLTAN_VERSION = current trilinos verson diff --git a/MAKE/Makefile.puhti_pgi b/MAKE/Makefile.puhti_pgi index 6048ef26d..59686cd4b 100644 --- a/MAKE/Makefile.puhti_pgi +++ b/MAKE/Makefile.puhti_pgi @@ -16,15 +16,29 @@ else VECTORCLASS = VEC4D_FALLBACK endif +FLAGS = + +#GNU flags: +CC_BRAND = pgi +CC_BRAND_VERSION = 19.7 +CXXFLAGS += -g -O3 -acc -D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL=1 -Minfo=accel +testpackage: CXXFLAGS = -g -O2 -acc -D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL=1 -Minfo=accel + +MATHFLAGS = +LDFLAGS = -O3 -acc -g +LIB_MPI = + #======== PAPI ========== #Add PAPI_MEM define to use papi to report memory consumption? #CXXFLAGS += -DPAPI_MEM +#testpackage: CXXFLAGS += -DPAPI_MEM #======== Allocator ========= #Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc #Configure jemalloc with --with-jemalloc-prefix=je_ when installing it #CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +#testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE #======= Compiler and compilation flags ========= @@ -39,17 +53,6 @@ endif # errors that come up when using # mpi.h in c++ on Cray -FLAGS = - -#GNU flags: -CC_BRAND = pgi -CC_BRAND_VERSION = 19.7 -CXXFLAGS += -g -O3 -acc -D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL=1 -Minfo=accel -testpackage: CXXFLAGS += -g -O2 -acc -D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL=1 -Minfo=accel - -MATHFLAGS = -LDFLAGS = -O3 -acc -g -LIB_MPI = # BOOST_VERSION = current trilinos version # ZOLTAN_VERSION = current trilinos verson diff --git a/MAKE/Makefile.sebpc b/MAKE/Makefile.sebpc deleted file mode 100644 index 561a74467..000000000 --- a/MAKE/Makefile.sebpc +++ /dev/null @@ -1,37 +0,0 @@ -CMP = mpic++ -LNK = mpic++ - -LIBRARY_PREFIX = $(HOME)/libraries -FLAGS = - - -CXXFLAGS = -I$(LIBRARY_PREFIX)/include -O3 -funroll-loops -std=c++0x -fopenmp -fabi-version=0 -mavx -MATHFLAGS = -ffast-math -LDFLAGS = -L$(LIBRARY_PREFIX)/lib -LIB_MPI = -lgomp - - -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen - -INC_BOOST = -LIB_BOOST = -lboost_program_options - - -LIB_ZOLTAN = -lzoltan - -INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv - -INC_SILO = -LIB_SILO = -lsilo - -INC_JEMALLOC = -LIB_JEMALLOC = -ljemalloc - -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg - -LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include - - -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass diff --git a/MAKE/Makefile.sisu_gcc b/MAKE/Makefile.sisu_gcc deleted file mode 100644 index bbfc2c8c5..000000000 --- a/MAKE/Makefile.sisu_gcc +++ /dev/null @@ -1,87 +0,0 @@ -CMP = CC -LNK = CC - -#======== Vectorization ========== -#Set vector backend type for vlasov solvers, sets precision and length. -#Options: -# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER -# AVX512: VEC8D_AGNER, VEC16F_AGNER -# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK - -ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) -#Single-precision - VECTORCLASS = VEC8F_AGNER -else -#Double-precision - VECTORCLASS = VEC4D_AGNER -endif - -#======== PAPI ========== -#Add PAPI_MEM define to use papi to report memory consumption? -CXXFLAGS += -DPAPI_MEM - - -#======== Allocator ========= -#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc -#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it -CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE - - -#======= Compiler and compilation flags ========= -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS += -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -#GNU flags: -CC_BRAND = gcc -CC_BRAND_VERSION = 6.2.0 -CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++11 -W -Wall -Wno-unused -fabi-version=0 -mavx2 -testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++0x -fabi-version=0 -mavx - -MATHFLAGS = -ffast-math -LDFLAGS = -LIB_MPI = -lgomp - -# BOOST_VERSION = current trilinos version -# ZOLTAN_VERSION = current trilinos verson - -#======== Libraries =========== - -MPT_VERSION = 7.5.1 -JEMALLOC_VERSION = 4.0.4 -LIBRARY_PREFIX = /proj/vlasov/libraries - - -#compiled libraries -INC_BOOST = -I$(CRAY_TRILINOS_PREFIX_DIR)/include/boost -LIB_BOOST = -L$(CRAY_TRILINOS_PREFIX_DIR)/lib -lboost_program_options - -INC_ZOLTAN = -I$(CRAY_TRILINOS_PREFIX_DIR)/include -LIB_ZOLTAN = -I$(CRAY_TRILINOS_PREFIX_DIR)/lib -lzoltan - -INC_JEMALLOC = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/include -LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/lib -ljemalloc - -INC_VLSV = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -lvlsv - -LIB_PROFILE = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/2.0/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/2.0/include - -#header libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen/ -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg_new_neighbours/ -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass -INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid diff --git a/MAKE/Makefile.sisu_intel b/MAKE/Makefile.sisu_intel deleted file mode 100644 index f59706474..000000000 --- a/MAKE/Makefile.sisu_intel +++ /dev/null @@ -1,81 +0,0 @@ -#Intel compile on sisu, Cray XC40 -#List of required module loads & swaps -#module swap PrgEnv-cray PrgEnv-intel -#module swap intel intel/15.0.2.164 -#module load cray-trilinos -#module load papi -#module load gcc/4.9.2 - - - -CMP = CC -LNK = CC - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = intel -CC_BRAND_VERSION = 14.0.3 -#With intel 15 these options are a few % faster than gnu 4.9 -CXXFLAGS += -Ofast -inline-level=2 -no-inline-min-size -std=c++11 -openmp -ansi-alias - -#CXXFLAGS += -O2 -std=c++11 -openmp -ansi-alias - - - - -MATHFLAGS = -LDFLAGS = -openmp -LIB_MPI = - -BOOST_VERSION = 1.54.0 -MPT_VERSION = 7.0.1 -ZOLTAN_VERSION = 3.8 -SILO_VERSION = 4.9.1 -JEMALLOC_VERSION = 3.6.0 -LIBRARY_PREFIX = /homeappl/home/alfthan/libraries - - -#compiled libraries -INC_BOOST = -I$(CRAY_TRILINOS_PREFIX_DIR)/include/boost -INC_BOOST = -L$(CRAY_TRILINOS_PREFIX_DIR)/lib -lboost_program_options - -INC_ZOLTAN = -I$(CRAY_TRILINOS_PREFIX_DIR)/include -LIB_ZOLTAN = -I$(CRAY_TRILINOS_PREFIX_DIR)/lib -lzoltan - - -INC_SILO = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/include -LIB_SILO = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/silo/$(SILO_VERSION)/lib -lsilo - -INC_JEMALLOC = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/include -LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/lib -ljemalloc - -INC_VLSV = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -lvlsv - - -LIB_PROFILE = -L$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX)/mpich2/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/include - -#header libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen/ -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/ -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass - - - - diff --git a/MAKE/Makefile.snbtest b/MAKE/Makefile.snbtest deleted file mode 100644 index 42d8542ff..000000000 --- a/MAKE/Makefile.snbtest +++ /dev/null @@ -1,27 +0,0 @@ -CMP = mpic++ -LNK = mpic++ - -FLAGS = -CXXFLAGS = -mavx -fabi-version=0 -I $(HOME)/include -L $(HOME)/lib -O3 -funroll-loops -std=c++0x -W -Wall -pedantic -Wno-unused -MATHFLAGS = -ffast-math -LDFLAGS = -L $(HOME)/lib -LIB_MPI = - -LIBRARY_PREFIX = $HOME -LIBRARY_PREFIX_B = $HOME - -INC_BOOST = -LIB_BOOST = -lboost_mpi -lboost_serialization -lboost_program_options - -INC_ZOLTAN = -LIB_ZOLTAN = -lzoltan - -INC_SILO = -LIB_SILO = -lsilo - -INC_DCCRG = - -LIB_PROFILE = -lphiprof -INC_PROFILE = -INC_TOPO = - diff --git a/MAKE/Makefile.taito_gcc b/MAKE/Makefile.taito_gcc deleted file mode 100644 index d9c8f684f..000000000 --- a/MAKE/Makefile.taito_gcc +++ /dev/null @@ -1,95 +0,0 @@ -CMP = mpic++ -LNK = mpic++ - -#======== Vectorization ========== -#Set vector backend type for vlasov solvers, sets precision and length. -#Options: -# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER -# AVX512: VEC8D_AGNER, VEC16F_AGNER -# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK - -ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) -#Single-precision - VECTORCLASS = VEC8F_AGNER -else -#Double-precision - VECTORCLASS = VEC4D_AGNER -endif - -#======== PAPI ========== -#Add PAPI_MEM define to use papi to report memory consumption? -#CXXFLAGS += -DPAPI_MEM - - -#======== Allocator ========= -#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc -#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it -CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE - - -#======= Compiler and compilation flags ========= -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -#GNU flags: -CC_BRAND = gcc -CC_BRAND_VERSION = 4.9.3 -CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++0x -W -Wall -Wno-unused -fabi-version=0 -mavx -testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++0x -fabi-version=0 -mavx - -MATHFLAGS = -ffast-math -LDFLAGS = -lrt -lgfortran -LIB_MPI = -lgomp - -# BOOST_VERSION = current trilinos version -# ZOLTAN_VERSION = current trilinos verson -# -#======== Libraries =========== - -MPT_VERSION = 3.1.3 -JEMALLOC_VERSION = 4.0.4 -LIBRARY_PREFIX = /proj/vlasov/libraries - - -#compiled libraries -INC_BOOST = -I$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/1.61.0/include/ -LIB_BOOST = -L$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/1.61.0/lib -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/3.83/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/3.83/lib -lzoltan - -INC_JEMALLOC = -I$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/include -LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/lib -ljemalloc -LDFLAGS += -Wl,-rpath=$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/lib - -INC_VLSV = -I$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -lvlsv - -LIB_PROFILE = -L$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/2.0/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/2.0/include -LDFLAGS += -Wl,-rpath=$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/2.0/lib - -#LIB_PAPI = -L$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/5.5.0/lib -lpapi -#INC_PAPI = -I$(LIBRARY_PREFIX)/taito/openmpi/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/5.5.0/include - -#header libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen/ -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/ -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass - - - - diff --git a/MAKE/Makefile.taito_gpu b/MAKE/Makefile.taito_gpu deleted file mode 100644 index b6d4522a7..000000000 --- a/MAKE/Makefile.taito_gpu +++ /dev/null @@ -1,65 +0,0 @@ -CMP = mpic++ -LNK = mpic++ - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = gcc -CC_BRAND_VERSION = 4.9.3 -CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++0x -W -Wall -Wno-unused-parameter -Wno-unused-variable -Wno-sign-compare -fabi-version=0 -mavx2 -testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++0x -fabi-version=0 -mavx - -MATHFLAGS = -ffast-math -LDFLAGS = -LIB_MPI = -lgomp -LIB_CUDA = -L$(CUDALIB) -lcudart -lstdc++ - -# Cuda -CUDACMP = nvcc -NVCCFLAGS = -O3 --restrict -gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 - -# BOOST_VERSION = current trilinos version -# ZOLTAN_VERSION = current trilinos verson - -MPT_VERSION = 1.10.2 -JEMALLOC_VERSION = 4.0.4 -LIBRARY_PREFIX = /wrk/trng114/libraries - - -#compiled libraries -INC_BOOST = -I$(LIBRARY_PREFIX)/boost -LIB_BOOST = -L$(LIBRARY_PREFIX)/boost/stage/lib -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/zoltan/lib -lzoltan - -INC_JEMALLOC = -I$(LIBRARY_PREFIX)/taito/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/include -LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/taito/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/$(JEMALLOC_VERSION)/lib -ljemalloc - -INC_VLSV = -I$(LIBRARY_PREFIX)/taito/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX)/taito/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -lvlsv - -LIB_PAPI = -L$(LIBRARY_PREFIX)/taito/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/lib -lpapi -INC_PAPI = -I$(LIBRARY_PREFIX)/taito/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/include - -LIB_PROFILE = -L$(LIBRARY_PREFIX)/taito/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib -lphiprof -lrt -INC_PROFILE = -I$(LIBRARY_PREFIX)/taito/$(MPT_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/include - -#header libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen/ -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/ -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass diff --git a/MAKE/Makefile.voima b/MAKE/Makefile.voima deleted file mode 100644 index ada2bc344..000000000 --- a/MAKE/Makefile.voima +++ /dev/null @@ -1,62 +0,0 @@ -CMP = CC -LNK = CC -not_parallel_tools: CMP = g++ -not_parallel_tools: LNK = g++ - -#-DNO_WRITE_AT_ALL: Define to disable write at all to -# avoid memleak (much slower IO) -#-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition -# errors that come up when using -# mpi.h in c++ on Cray - -CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK -DEIGEN_NO_DEBUG - -FLAGS = - -# NOTES on compiler flags: -# CXXFLAGS is for compiler flags, they are always used -# MATHFLAGS are for special math etc. flags, these are only applied on solver functions -# LDFLAGS flags for linker - -#GNU flags: -CC_BRAND = gcc -CC_BRAND_VERSION = 4.8.2 -#CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++0x -W -Wall -Wno-unused -fabi-version=0 -mavx -CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++0x -fabi-version=0 -mavx -testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++0x -fabi-version=0 -mavx - - -MATHFLAGS = -ffast-math -LDFLAGS = -LIB_MPI = -lgomp - -BOOST_VERSION = 1.54.0 -CRAY_MPICH_VERSION = 6.1.0 -ZOLTAN_VERSION = 3.6 -SILO_VERSION = 4.9.1 - - -LIBRARY_PREFIX = /stornext/field/users/alfthan/libraries -LIBRARY_PREFIX_COMPILED = $(LIBRARY_PREFIX)/cray-mpich/$(CRAY_MPICH_VERSION)/$(CC_BRAND)/$(CC_BRAND_VERSION) - -#compiled libraries - -INC_VLSV = -I$(LIBRARY_PREFIX_COMPILED)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX_COMPILED)/vlsv -lvlsv -INC_JEMALLOC = -I$(LIBRARY_PREFIX_COMPILED)/jemalloc/3.6.0/include -LIB_JEMALLOC = -L$(LIBRARY_PREFIX_COMPILED)/jemalloc/3.6.0/lib -ljemalloc -INC_BOOST = -I$(LIBRARY_PREFIX_COMPILED)/boost/$(BOOST_VERSION)/include -LIB_BOOST = -L$(LIBRARY_PREFIX_COMPILED)/boost/$(BOOST_VERSION)/lib -lboost_program_options -INC_ZOLTAN = -I$(LIBRARY_PREFIX_COMPILED)/zoltan/$(ZOLTAN_VERSION)/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX_COMPILED)/zoltan/$(ZOLTAN_VERSION)/lib -lzoltan -LIB_PROFILE = -L$(LIBRARY_PREFIX_COMPILED)/phiprof2/lib -lphiprof -INC_PROFILE = -I$(LIBRARY_PREFIX_COMPILED)/phiprof2/include -INC_SILO = -I$(LIBRARY_PREFIX_COMPILED)/silo/$(SILO_VERSION)/include -LIB_SILO = -L$(LIBRARY_PREFIX_COMPILED)/silo/$(SILO_VERSION)/lib -lsilo -INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid - -#header libraries - -INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen/eigen-eigen-2249f9c22fe8 -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg -INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass diff --git a/MAKE/Makefile.vorna_gcc b/MAKE/Makefile.vorna_gcc new file mode 100644 index 000000000..ab21c376c --- /dev/null +++ b/MAKE/Makefile.vorna_gcc @@ -0,0 +1,87 @@ +CMP = mpic++ +LNK = mpic++ + +#======== Vectorization ========== +#Set vector backend type for vlasov solvers, sets precision and length. +#Options: +# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER +# AVX512: VEC8D_AGNER, VEC16F_AGNER +# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK + +ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) +#Single-precision + VECTORCLASS = VEC8F_AGNER +else +#Double-precision + VECTORCLASS = VEC4D_AGNER +endif + +FLAGS = + +#GNU flags: +CC_BRAND = gcc +CC_BRAND_VERSION = 8.3.0 +CXXFLAGS += -O3 -fopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -mavx +testpackage: CXXFLAGS = -O2 -fopenmp -funroll-loops -std=c++17 -mavx + +MATHFLAGS = -ffast-math +LDFLAGS = -lrt -lgfortran -std=c++17 +LIB_MPI = -lgomp + +#======== PAPI ========== +#Add PAPI_MEM define to use papi to report memory consumption? +CXXFLAGS += -DPAPI_MEM +testpackage: CXXFLAGS += -DPAPI_MEM + +#======== Allocator ========= +#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc +#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it +CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE + +#======= Compiler and compilation flags ========= +# NOTES on compiler flags: +# CXXFLAGS is for compiler flags, they are always used +# MATHFLAGS are for special math etc. flags, these are only applied on solver functions +# LDFLAGS flags for linker + +# BOOST_VERSION = current trilinos version +# ZOLTAN_VERSION = current trilinos verson +# +#======== Libraries =========== + +MPT_VERSION = 3.1.3 +JEMALLOC_VERSION = 4.0.4 +#LIBRARY_PREFIX = /proj/markusb/libraries +LIBRARY_PREFIX = /proj/group/spacephysics/libraries + + +#compiled libraries +INC_BOOST = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/include +LIB_BOOST = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/lib -lboost_program_options + +INC_ZOLTAN = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/include +LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/lib -lzoltan + +INC_JEMALLOC = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/include +LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/lib -ljemalloc + +INC_VLSV = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv +LIB_VLSV = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv -lvlsv + +LIB_PROFILE = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib -lphiprof +INC_PROFILE = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/include + +LDFLAGS += -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/phiprof/lib:$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/boost/lib:$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/vlsv/lib:$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/jemalloc/lib:$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/zoltan/lib + +LIB_PAPI = -L$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/lib -lpapi -Wl,-rpath=$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/lib +INC_PAPI = -I$(LIBRARY_PREFIX)/$(CC_BRAND)/$(CC_BRAND_VERSION)/papi/include + +#header libraries +INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid/ +INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg/ +INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/vectorclass/ + + + + diff --git a/MAKE/Makefile.vorna_intel b/MAKE/Makefile.vorna_intel index 0638b02c8..70a1b4f2a 100644 --- a/MAKE/Makefile.vorna_intel +++ b/MAKE/Makefile.vorna_intel @@ -16,16 +16,17 @@ else VECTORCLASS = VEC4D_AGNER endif -#======== PAPI ========== -#Add PAPI_MEM define to use papi to report memory consumption? -#CXXFLAGS += -DPAPI_MEM - +FLAGS = -#======== Allocator ========= -#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc -#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it -CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +#GNU flags: +CC_BRAND = gcc +CC_BRAND_VERSION = 7.3.0 +CXXFLAGS += -g -O3 -qopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -mavx +testpackage: CXXFLAGS += -g -O2 -qopenmp -funroll-loops -std=c++17 -mavx +MATHFLAGS = -ffast-math +LDFLAGS = -lrt -std=c++17 -liomp5 +LIB_MPI = -lgomp #======= Compiler and compilation flags ========= # NOTES on compiler flags: @@ -40,18 +41,19 @@ CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE # mpi.h in c++ on Cray CXXFLAGS += -DMPICH_IGNORE_CXX_SEEK +testpackage: CXXFLAGS += -DMPICH_IGNORE_CXX_SEEK -FLAGS = +#======== PAPI ========== +#Add PAPI_MEM define to use papi to report memory consumption? +#CXXFLAGS += -DPAPI_MEM +#testpackage: CXXFLAGS += -DPAPI_MEM -#GNU flags: -CC_BRAND = gcc -CC_BRAND_VERSION = 7.3.0 -CXXFLAGS += -g -O3 -qopenmp -funroll-loops -std=c++17 -W -Wall -Wno-unused -mavx -testpackage: CXXFLAGS += -g -O2 -qopenmp -funroll-loops -std=c++0x -mavx +#======== Allocator ========= +#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc +#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it +CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE -MATHFLAGS = -ffast-math -LDFLAGS = -lrt -std=c++17 -liomp5 -LIB_MPI = -lgomp # BOOST_VERSION = current trilinos version # ZOLTAN_VERSION = current trilinos verson diff --git a/MAKE/Makefile.yann b/MAKE/Makefile.yann index f076db91b..4beabb485 100644 --- a/MAKE/Makefile.yann +++ b/MAKE/Makefile.yann @@ -10,9 +10,11 @@ ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) #Single-precision + INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/Codes/vectorclass/version1 VECTORCLASS = VEC4F_FALLBACK else #Double-precision + INC_VECTORCLASS = -I$(LIBRARY_PREFIX)/Codes/vectorclass/version1 VECTORCLASS = VEC4D_FALLBACK endif @@ -25,22 +27,6 @@ LNK = mpic++ PAPI_FLAG = -#======== Vectorization ========== -#Set vector backend type for vlasov solvers, sets precision and length. -#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) -#Options: -# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER -# AVX512: VEC8D_AGNER, VEC16F_AGNER -# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK - -ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) -#Single-precision - VECTORCLASS = VEC8F_AGNER -else -#Double-precision - VECTORCLASS = VEC4D_AGNER -endif - FLAGS = #CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -g -funroll-loops -std=c++0x -fopenmp -W -Wall -pedantic -Wno-unused -fabi-version=0 -mavx CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -O3 -funroll-loops -std=c++0x -fopenmp -W -Wall -Wno-unused -fabi-version=0 -mavx @@ -65,7 +51,7 @@ LIB_VLSV = -L$(LIBRARY_PREFIX)/Development/vlsv -lvlsv INC_SILO = -I$(LIBRARY_PREFIX)/include LIB_SILO = -L$(LIBRARY_PREFIX)/lib -lsilo -INC_DCCRG = -I$(LIBRARY_PREFIX)/Codes/dccrg +INC_DCCRG = -I$(LIBRARY_PREFIX)/Development/dccrg INC_FSGRID = -I$(LIBRARY_PREFIX)/Development/fsgrid diff --git a/Makefile b/Makefile index 4a1c53ded..2dcbe7c99 100644 --- a/Makefile +++ b/Makefile @@ -346,7 +346,7 @@ Shock.o: ${DEPS_COMMON} projects/Shock/Shock.h projects/Shock/Shock.cpp ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c projects/Shock/Shock.cpp ${INC_DCCRG} ${INC_ZOLTAN} ${INC_BOOST} ${INC_EIGEN} ${INC_FSGRID} IPShock.o: ${DEPS_COMMON} projects/IPShock/IPShock.h projects/IPShock/IPShock.cpp - ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c projects/IPShock/IPShock.cpp ${INC_DCCRG} ${INC_ZOLTAN} ${INC_BOOST} ${INC_EIGEN} ${INC_FSGRID} ${INC_VECTORCLASS} + ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c projects/IPShock/IPShock.cpp ${INC_DCCRG} ${INC_ZOLTAN} ${INC_BOOST} ${INC_EIGEN} ${INC_FSGRID} Template.o: ${DEPS_COMMON} projects/Template/Template.h projects/Template/Template.cpp ${CMP} ${CXXFLAGS} ${FLAGS} ${MATHFLAGS} -c projects/Template/Template.cpp ${INC_DCCRG} ${INC_ZOLTAN} ${INC_BOOST} ${INC_EIGEN} ${INC_FSGRID} diff --git a/README.md b/README.md new file mode 100644 index 000000000..179ba3335 --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ + +![](https://github.com/fmihpc/vlasiator/blob/master/doc/artwork/logo_black.png?raw=true) + +Vlasiator - ten letters you can count on +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3640594.svg)](https://doi.org/10.5281/zenodo.3640594) + +## Code description +Space weather is a term used to describe the variable environmental effects within near-Earth space, caused by the Sun emitting solar wind, a stream of charged particles carrying the solar electromagnetic field. Space weather can be caused by solar high-energy particles or by dynamic variations of the solar wind that can cause extended periods of major disturbances on ground and space, affecting technological systems (e.g., telecommunication and weather spacecraft at geostationary orbit, and ground-based power grids). + +In Vlasiator, ions are represented as velocity distribution functions, while electrons are magnetohydrodynamic fluid, enabling a self-consistent global plasma simulation that can describe multi-temperature plasmas to resolve non-MHD processes that currently cannot be self-consistently described by the existing global space weather simulations. The novelty is that by modelling ions as velocity distribution functions the outcome will be numerically noiseless. + +Due to the multi-dimensional approach at ion scales, Vlasiator's computational challenges are immense. We use advanced high performance computing techniques to allow massively parallel computations on tens of thousands of cores. + +## Documentation +See the [wiki](https://github.com/fmihpc/vlasiator/wiki) for build instructions and general advice. + +## Paper references +- [Palmroth, M., Ganse, U., Pfau-Kempf, Y., Battarbee, M., Turc, L., Brito, T., Grandin, M., Hoilijoki, S., Sandroos, A. & von Alfthan, S. (2018). Vlasov methods in space physics and astrophysics. Living Reviews in Computational Astrophysics, 4, 1. doi: 10.1007/s41115-018-0003-2](https://link.springer.com/article/10.1007/s41115-018-0003-2) +- [von Alfthan, S., Pokhotelov, D., Kempf, Y., Hoilijoki, S., Honkonen, I., Sandroos, A. & Palmroth, M. (2014). Vlasiator: First global hybrid-Vlasov simulations of Earth's foreshock and magnetosheath . Journal of Atmospheric and Solar-Terrestrial Physics , 120, 24 - 35. doi: http://dx.doi.org/10.1016/j.jastp.2014.08.012](http://www.sciencedirect.com/science/article/pii/S1364682614001916) diff --git a/backgroundfield/backgroundfield.cpp b/backgroundfield/backgroundfield.cpp index ac4c0baf1..8f4efd54d 100644 --- a/backgroundfield/backgroundfield.cpp +++ b/backgroundfield/backgroundfield.cpp @@ -31,7 +31,7 @@ //FieldFunction should be initialized void setBackgroundField( FieldFunction& bgFunction, - FsGrid< std::array, 2>& BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, bool append) { /*if we do not add a new background to the existing one we first put everything to zero*/ @@ -134,7 +134,7 @@ void setBackgroundField( } void setBackgroundFieldToZero( - FsGrid< std::array, 2>& BgBGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid ) { auto localSize = BgBGrid.getLocalSize().data(); @@ -153,7 +153,7 @@ void setBackgroundFieldToZero( void setPerturbedField( FieldFunction& bfFunction, - FsGrid< std::array, 2>& perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, bool append) { /*if we do not add a new background to the existing one we first put everything to zero*/ @@ -219,7 +219,7 @@ void setPerturbedField( } void setPerturbedFieldToZero( - FsGrid< std::array, 2> & perBGrid) { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid) { auto localSize = perBGrid.getLocalSize().data(); #pragma omp parallel for collapse(3) diff --git a/backgroundfield/backgroundfield.h b/backgroundfield/backgroundfield.h index 0c880cef3..6aac97e8a 100644 --- a/backgroundfield/backgroundfield.h +++ b/backgroundfield/backgroundfield.h @@ -30,22 +30,22 @@ void setBackgroundField( FieldFunction& bgFunction, - FsGrid< std::array, 2>& BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, bool append=false ); void setBackgroundFieldToZero( - FsGrid< std::array, 2>& BgBGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid ); void setPerturbedField( FieldFunction& bgFunction, - FsGrid< std::array, 2>& perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, bool append=false ); void setPerturbedFieldToZero( - FsGrid< std::array, 2>& perBGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid ); #endif diff --git a/common.h b/common.h index 3426c2427..3e84ca069 100644 --- a/common.h +++ b/common.h @@ -206,19 +206,19 @@ namespace bvolderivatives { }; } -// FsGrid< std::array, 2> & perBGrid, -// FsGrid< std::array, 2> & perBDt2Grid, -// FsGrid< std::array, 2> & EGrid, -// FsGrid< std::array, 2> & EDt2Grid, -// FsGrid< std::array, 2> & EHallGrid, -// FsGrid< std::array, 2> & EGradPeGrid, -// FsGrid< std::array, 2> & momentsGrid, -// FsGrid< std::array, 2> & momentsDt2Grid, -// FsGrid< std::array, 2> & dPerBGrid, -// FsGrid< std::array, 2> & dMomentsGrid, -// FsGrid< std::array, 2> & BgBGrid, -// FsGrid< std::array, 2> & volGrid, -// FsGrid< fsgrids::technical, 2> & technicalGrid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & EDt2Grid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, +// FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, +// FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, /*! Namespace containing enums and structs for the various field solver grid instances * @@ -320,6 +320,9 @@ namespace fsgrids { dVzdx, /*!< Derivative of volume-averaged Vz to x-direction. */ dVzdy, /*!< Derivative of volume-averaged Vz to y-direction. */ dVzdz, /*!< Derivative of volume-averaged Vz to z-direction. */ + dPedx, /*!< Derivative of electron pressure to x-direction. */ + dPedy, /*!< Derivative of electron pressure to y-direction. */ + dPedz, /*!< Derivative of electron pressure to z-direction. */ N_DMOMENTS }; @@ -369,6 +372,7 @@ namespace fsgrids { Real maxFsDt; /*!< maximum timestep allowed in ordinary space by fieldsolver for this cell**/ int fsGridRank; /*!< Rank in the fsGrids cartesian coordinator */ uint SOLVE; /*!< Bit mask to determine whether a given cell should solve E or B components. */ + int refLevel; /*! using namespace std; void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosticReducer) @@ -44,20 +43,21 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti */ // Sidestep mixed case errors - const std::string lowercase = boost::algorithm::to_lower_copy(*it); + std::string lowercase = *it; + for(auto& c : lowercase) c = tolower(c); if(lowercase == "fg_b" || lowercase == "b") { // Bulk magnetic field at Yee-Lattice locations outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_b",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]*3); @@ -83,16 +83,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } if(lowercase == "fg_backgroundb" || lowercase == "backgroundb" || lowercase == "fg_b_background") { // Static (typically dipole) magnetic field part outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_b_background",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]*3); @@ -115,16 +115,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } if(lowercase == "fg_perturbedb" || lowercase == "perturbedb" || lowercase == "fg_b_perturbed") { // Fluctuating magnetic field part outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_b_perturbed",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]*3); @@ -142,21 +142,21 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti return retval; } )); - outputReducer->addMetadata(outputReducer->size()-1,"T","$\\mathrm{T}$","$B_\\mathrm{per}$)","1.0"); + outputReducer->addMetadata(outputReducer->size()-1,"T","$\\mathrm{T}$","$B_\\mathrm{per}$","1.0"); continue; } if(lowercase == "fg_e" || lowercase == "e") { // Bulk electric field at Yee-lattice locations outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_e",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]*3); @@ -184,16 +184,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } if(lowercase == "fg_rhom") { // Overall mass density (summed over all populations) outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_rhom",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]); @@ -219,16 +219,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } if(lowercase == "fg_rhoq") { // Overall charge density (summed over all populations) outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_rhoq",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]); @@ -264,16 +264,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } if(lowercase == "fg_v") { // Overall effective bulk density defining the center-of-mass frame from all populations outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_v",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]*3); @@ -418,16 +418,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti if(lowercase == "maxfieldsdt" || lowercase == "fg_maxfieldsdt" || lowercase == "fg_maxdt_fieldsolver") { // Maximum timestep constraint as calculated by the fieldsolver outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_maxdt_fieldsolver",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]); @@ -455,16 +455,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti if(lowercase == "fsgridrank" || lowercase == "fg_rank") { // Map of spatial decomposition of the FsGrid into MPI ranks outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_rank",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2],technicalGrid.getRank()); @@ -474,6 +474,38 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti outputReducer->addMetadata(outputReducer->size()-1,"","","$\\mathrm{fGrid rank}$",""); continue; } + if(lowercase == "fg_amr_level") { + // Map of spatial decomposition of the FsGrid into MPI ranks + outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_amr_level",[]( + FsGrid< std::array, FS_STENCIL_WIDTH>& perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH>& EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH>& EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH>& EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH>& momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH>& dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH>& dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH>& BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH>& volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH>& technicalGrid)->std::vector { + + + std::array& gridSize = technicalGrid.getLocalSize(); + std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]); + + // Iterate through fsgrid cells and extract corresponding AMR level + for(int z=0; zrefLevel; + } + } + } + return retval; + } + )); + outputReducer->addMetadata(outputReducer->size()-1,"","","$\\mathrm{fGrid rank}$",""); + continue; + } if(lowercase == "boundarytype" || lowercase == "vg_boundarytype") { // Type of boundarycells outputReducer->addOperator(new DRO::BoundaryType); @@ -483,16 +515,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti if(lowercase == "fsgridboundarytype" || lowercase == "fg_boundarytype") { // Type of boundarycells as stored in FSGrid outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_boundarytype",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]); @@ -520,16 +552,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti if(lowercase == "fsgridboundarylayer" || lowercase == "fg_boundarylayer") { // Type of boundarycells as stored in FSGrid outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_boundarylayer",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]); @@ -582,16 +614,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } if(lowercase == "fg_vole" || lowercase == "fg_e_vol" || lowercase == "fg_evol") { outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_e_vol",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]*3); @@ -616,16 +648,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti for(int index=0; indexaddOperator(new DRO::DataReductionOperatorFsGrid(reducer_name,[index]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]); @@ -648,7 +680,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti if(lowercase =="gradpee" || lowercase == "e_gradpe" || lowercase == "vg_e_gradpe") { // Electron pressure gradient contribution to the generalized ohm's law outputReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_e_gradpe",CellParams::EXGRADPE,3)); - outputReducer->addMetadata(outputReducer->size()-1,"V/m","$\\mathrm{V}\\,\\mathrm{m}^{-1}$","$E_{\\del P_\\mathrm{e}}$","1.0"); + outputReducer->addMetadata(outputReducer->size()-1,"V/m","$\\mathrm{V}\\,\\mathrm{m}^{-1}$","$E_{\\nabla P_\\mathrm{e}}$","1.0"); continue; } if(lowercase == "volb" || lowercase == "vg_volb" || lowercase == "b_vol" || lowercase == "bvol" || lowercase == "vg_bvol" || lowercase == "vg_b_vol") { @@ -659,16 +691,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } if(lowercase == "fg_volb" || lowercase == "fg_bvol" || lowercase == "fg_b_vol") { // Static (typically dipole) magnetic field part outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_b_vol",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]*3); @@ -711,16 +743,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti if(lowercase == "fg_pressure") { // Overall scalar pressure from all populations outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_pressure",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]); @@ -786,16 +818,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } if(lowercase == "fg_gridcoordinates") { outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_x",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]); @@ -813,16 +845,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti )); outputReducer->addMetadata(outputReducer->size()-1,"m","$\\mathrm{m}$","$X_\\mathrm{fg}$","1.0"); outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_y",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]); @@ -840,16 +872,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti )); outputReducer->addMetadata(outputReducer->size()-1,"m","$\\mathrm{m}$","$Y_\\mathrm{fg}$","1.0"); outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_z",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]); @@ -867,16 +899,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti )); outputReducer->addMetadata(outputReducer->size()-1,"m","$\\mathrm{m}$","$Z_\\mathrm{fg}$","1.0"); outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_dx",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2], technicalGrid.DX); @@ -885,16 +917,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti )); outputReducer->addMetadata(outputReducer->size()-1,"m","$\\mathrm{m}$","$\\delta X_\\mathrm{fg}$","1.0"); outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_dy",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2], technicalGrid.DY); @@ -903,16 +935,16 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti )); outputReducer->addMetadata(outputReducer->size()-1,"m","$\\mathrm{m}$","$\\delta Y_\\mathrm{fg}$","1.0"); outputReducer->addOperator(new DRO::DataReductionOperatorFsGrid("fg_dz",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2], technicalGrid.DZ); @@ -942,7 +974,8 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti it++) { // Sidestep mixed case errors - const std::string lowercase = boost::algorithm::to_lower_copy(*it); + std::string lowercase = *it; + for(auto& c : lowercase) c = tolower(c); if(lowercase == "populations_blocks" || lowercase == "populations_vg_blocks") { // Per-population total block counts @@ -953,7 +986,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti } if(lowercase == "vg_rhom" || lowercase == "rhom") { // Overall mass density - diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("rhom",CellParams::RHOM,1)); + diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_rhom",CellParams::RHOM,1)); continue; } if(lowercase == "populations_rholossadjust" || lowercase == "populations_rho_loss_adjust" || lowercase == "populations_vg_rho_loss_adjust") { @@ -961,7 +994,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti for(unsigned int i =0; i < getObjectWrapper().particleSpecies.size(); i++) { species::Species& species=getObjectWrapper().particleSpecies[i]; const std::string& pop = species.name; - diagnosticReducer->addOperator(new DRO::DataReductionOperatorPopulations(pop + "/rho_loss_adjust", i, offsetof(spatial_cell::Population, RHOLOSSADJUST), 1)); + diagnosticReducer->addOperator(new DRO::DataReductionOperatorPopulations(pop + "/vg_rho_loss_adjust", i, offsetof(spatial_cell::Population, RHOLOSSADJUST), 1)); } continue; } @@ -970,28 +1003,28 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti // continue; //} if(lowercase == "lbweight" || lowercase == "vg_lbweight" || lowercase == "vg_loadbalanceweight" || lowercase == "vg_loadbalance_weight" || lowercase == "loadbalance_weight") { - diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("loadbalance_weight",CellParams::LBWEIGHTCOUNTER,1)); + diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_loadbalance_weight",CellParams::LBWEIGHTCOUNTER,1)); continue; } if(lowercase == "maxvdt" || lowercase == "maxdt_acceleration" || lowercase == "vg_maxdt_acceleration") { - diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("maxdt_acceleration",CellParams::MAXVDT,1)); + diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_maxdt_acceleration",CellParams::MAXVDT,1)); continue; } if(lowercase == "maxrdt" || lowercase == "maxdt_translation" || lowercase == "vg_maxdt_translation") { - diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("maxdt_translation",CellParams::MAXRDT,1)); + diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("vg_maxdt_translation",CellParams::MAXRDT,1)); continue; } if(lowercase == "maxfieldsdt" || lowercase == "maxdt_fieldsolver" || lowercase == "fg_maxfieldsdt" || lowercase == "fg_maxdt_fieldsolver") { - diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("maxdt_fieldsolver",CellParams::MAXFDT,1)); + diagnosticReducer->addOperator(new DRO::DataReductionOperatorCellParams("fg_maxdt_fieldsolver",CellParams::MAXFDT,1)); continue; } - if(lowercase == "populations_maxdistributionfunction") { + if(lowercase == "populations_maxdistributionfunction" || lowercase == "populations_vg_maxdistributionfunction") { for(unsigned int i =0; i < getObjectWrapper().particleSpecies.size(); i++) { diagnosticReducer->addOperator(new DRO::MaxDistributionFunction(i)); } continue; } - if(lowercase == "populations_mindistributionfunction") { + if(lowercase == "populations_mindistributionfunction" || lowercase == "populations_vg_mindistributionfunction") { for(unsigned int i =0; i < getObjectWrapper().particleSpecies.size(); i++) { diagnosticReducer->addOperator(new DRO::MinDistributionFunction(i)); } @@ -1001,7 +1034,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti for(unsigned int i =0; i < getObjectWrapper().particleSpecies.size(); i++) { species::Species& species=getObjectWrapper().particleSpecies[i]; const std::string& pop = species.name; - diagnosticReducer->addOperator(new DRO::DataReductionOperatorPopulations(pop + "/maxdt_translation", i, offsetof(spatial_cell::Population, max_dt[0]), 1)); + diagnosticReducer->addOperator(new DRO::DataReductionOperatorPopulations(pop + "/vg_maxdt_translation", i, offsetof(spatial_cell::Population, max_dt[0]), 1)); } continue; } @@ -1009,7 +1042,7 @@ void initializeDataReducers(DataReducer * outputReducer, DataReducer * diagnosti for(unsigned int i =0; i < getObjectWrapper().particleSpecies.size(); i++) { species::Species& species=getObjectWrapper().particleSpecies[i]; const std::string& pop = species.name; - diagnosticReducer->addOperator(new DRO::DataReductionOperatorPopulations(pop + "/maxdt_acceleration", i, offsetof(spatial_cell::Population, max_dt[1]), 1)); + diagnosticReducer->addOperator(new DRO::DataReductionOperatorPopulations(pop + "/vg_maxdt_acceleration", i, offsetof(spatial_cell::Population, max_dt[1]), 1)); } continue; } @@ -1187,22 +1220,25 @@ bool DataReducer::writeParameters(const unsigned int& operatorID, vlsv::Writer& /** Write all data thet the given DataReductionOperator wants to obtain from fsgrid into the output file. */ bool DataReducer::writeFsGridData( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, const std::string& meshName, const unsigned int operatorID, vlsv::Writer& vlsvWriter) { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, + const std::string& meshName, const unsigned int operatorID, + vlsv::Writer& vlsvWriter, + const bool writeAsFloat) { if (operatorID >= operators.size()) return false; DRO::DataReductionOperatorFsGrid* DROf = dynamic_cast(operators[operatorID]); if(!DROf) { return false; } else { - return DROf->writeFsGridData(perBGrid, EGrid, EHallGrid, EGradPeGrid, momentsGrid, dPerBGrid, dMomentsGrid, BgBGrid, volGrid, technicalGrid, meshName, vlsvWriter); + return DROf->writeFsGridData(perBGrid, EGrid, EHallGrid, EGradPeGrid, momentsGrid, dPerBGrid, dMomentsGrid, BgBGrid, volGrid, technicalGrid, meshName, vlsvWriter, writeAsFloat); } } diff --git a/datareduction/datareducer.h b/datareduction/datareducer.h index a8816e079..33225e663 100644 --- a/datareduction/datareducer.h +++ b/datareduction/datareducer.h @@ -26,6 +26,7 @@ #include #include "fsgrid.hpp" +#include "../definitions.h" #include "../spatial_cell.hpp" #include "datareductionoperator.h" @@ -59,16 +60,19 @@ class DataReducer { vlsv::Writer& vlsvWriter); bool writeParameters(const unsigned int& operatorID, vlsv::Writer& vlsvWriter); bool writeFsGridData( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, const std::string& meshName, const unsigned int operatorID, vlsv::Writer& vlsvWriter); + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, + const std::string& meshName, const unsigned int operatorID, + vlsv::Writer& vlsvWriter, + const bool writeAsFloat = false); private: /** Private copy-constructor to prevent copying the class. diff --git a/datareduction/datareductionoperator.cpp b/datareduction/datareductionoperator.cpp index b29238d08..5f815b12d 100644 --- a/datareduction/datareductionoperator.cpp +++ b/datareduction/datareductionoperator.cpp @@ -113,6 +113,7 @@ namespace DRO { std::string DataReductionOperatorFsGrid::getName() const {return variableName;} bool DataReductionOperatorFsGrid::getDataVectorInfo(std::string& dataType,unsigned int& dataSize,unsigned int& vectorSize) const { + // These are only set to dmmy values, as this reducer writes its own vlsv dataset anyway dataType = "float"; dataSize = sizeof(double); vectorSize = 1; @@ -130,16 +131,18 @@ namespace DRO { } bool DataReductionOperatorFsGrid::writeFsGridData( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, const std::string& meshName, vlsv::Writer& vlsvWriter) { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, + const std::string& meshName, vlsv::Writer& vlsvWriter, + const bool writeAsFloat) { std::map attribs; attribs["mesh"]=meshName; @@ -151,12 +154,26 @@ namespace DRO { std::vector varBuffer = lambda(perBGrid,EGrid,EHallGrid,EGradPeGrid,momentsGrid,dPerBGrid,dMomentsGrid,BgBGrid,volGrid,technicalGrid); - + std::array& gridSize = technicalGrid.getLocalSize(); int vectorSize = varBuffer.size() / (gridSize[0]*gridSize[1]*gridSize[2]); - if(vlsvWriter.writeArray("VARIABLE",attribs, "float", gridSize[0]*gridSize[1]*gridSize[2], vectorSize, sizeof(double), reinterpret_cast(varBuffer.data())) == false) { - string message = "The DataReductionOperator " + this->getName() + " failed to write its data."; - bailout(true, message, __FILE__, __LINE__); + + if(writeAsFloat) { + // Convert down to 32bit floats to save output space + std::vector varBufferFloat(varBuffer.size()); + for(uint i=0; i(varBufferFloat.data())) == false) { + string message = "The DataReductionOperator " + this->getName() + " failed to write its data."; + bailout(true, message, __FILE__, __LINE__); + } + + } else { + if(vlsvWriter.writeArray("VARIABLE",attribs, "float", gridSize[0]*gridSize[1]*gridSize[2], vectorSize, sizeof(double), reinterpret_cast(varBuffer.data())) == false) { + string message = "The DataReductionOperator " + this->getName() + " failed to write its data."; + bailout(true, message, __FILE__, __LINE__); + } } return true; @@ -498,7 +515,7 @@ namespace DRO { } MaxDistributionFunction::~MaxDistributionFunction() { } - std::string MaxDistributionFunction::getName() const {return popName + "/maximumdistributionfunctionvalue";} + std::string MaxDistributionFunction::getName() const {return popName + "/vg_maxdistributionfunction";} bool MaxDistributionFunction::getDataVectorInfo(std::string& dataType,unsigned int& dataSize,unsigned int& vectorSize) const { dataType = "float"; @@ -552,7 +569,7 @@ namespace DRO { } MinDistributionFunction::~MinDistributionFunction() { } - std::string MinDistributionFunction::getName() const {return popName + "/minimumdistributionfunctionvalue";} + std::string MinDistributionFunction::getName() const {return popName + "/vg_mindistributionfunction";} bool MinDistributionFunction::getDataVectorInfo(std::string& dataType,unsigned int& dataSize,unsigned int& vectorSize) const { dataType = "float"; @@ -1338,7 +1355,7 @@ namespace DRO { emax = getObjectWrapper().particleSpecies[popID].precipitationEmax; // already converted to SI nChannels = getObjectWrapper().particleSpecies[popID].precipitationNChannels; // number of energy channels, logarithmically spaced between emin and emax for (int i=0; i( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)> ReductionLambda; + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)> ReductionLambda; private: ReductionLambda lambda; std::string variableName; @@ -125,16 +125,18 @@ namespace DRO { virtual bool reduceData(const SpatialCell* cell,char* buffer); virtual bool reduceDiagnostic(const SpatialCell* cell,Real * result); virtual bool writeFsGridData( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, const std::string& meshName, vlsv::Writer& vlsvWriter); + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, + const std::string& meshName, vlsv::Writer& vlsvWriter, + const bool writeAsFloat=false); }; class DataReductionOperatorCellParams: public DataReductionOperator { diff --git a/definitions.h b/definitions.h index c8d20c114..4667148d3 100644 --- a/definitions.h +++ b/definitions.h @@ -93,7 +93,7 @@ typedef Realf (*AmrVelRefinement)(const Realf* velBlock); // neighborhoods, these are initialized in grid.cpp:initializeGrid -#define FIELD_SOLVER_NEIGHBORHOOD_ID 1 +//#define FIELD_SOLVER_NEIGHBORHOOD_ID 1 // Now fields on FSgrid #define VLASOV_SOLVER_NEIGHBORHOOD_ID 2 //up to third(PPM) neighbor in each face direction #define VLASOV_SOLVER_X_NEIGHBORHOOD_ID 3 //up to third(PPM) neighbor in x face directions #define VLASOV_SOLVER_Y_NEIGHBORHOOD_ID 4 //up to third(PPM) neighbor in y face directions diff --git a/fieldsolver/derivatives.cpp b/fieldsolver/derivatives.cpp index 5aa6f45c3..4660d7924 100644 --- a/fieldsolver/derivatives.cpp +++ b/fieldsolver/derivatives.cpp @@ -44,11 +44,11 @@ void calculateDerivatives( cint i, cint j, cint k, - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, cint& RKCase ) { @@ -58,7 +58,12 @@ void calculateDerivatives( // Get boundary flag for the cell: cuint sysBoundaryFlag = technicalGrid.get(i,j,k)->sysBoundaryFlag; cuint sysBoundaryLayer = technicalGrid.get(i,j,k)->sysBoundaryLayer; - + + // Constants for electron pressure derivatives + // Upstream pressure + Real Peupstream = Parameters::electronTemperature * Parameters::electronDensity * physicalconstants::K_B; + Real Peconst = Peupstream * pow(Parameters::electronDensity, -Parameters::electronPTindex); + std::array * leftMoments = NULL; std::array * leftPerB = NULL; std::array * centMoments = momentsGrid.get(i,j,k); @@ -111,6 +116,10 @@ void calculateDerivatives( dMoments->at(fsgrids::dmoments::dVzdx) = limiter(leftMoments->at(fsgrids::moments::VZ), centMoments->at(fsgrids::moments::VZ), rghtMoments->at(fsgrids::moments::VZ)); dPerB->at(fsgrids::dperb::dPERBydx) = limiter(leftPerB->at(fsgrids::bfield::PERBY),centPerB->at(fsgrids::bfield::PERBY),rghtPerB->at(fsgrids::bfield::PERBY)); dPerB->at(fsgrids::dperb::dPERBzdx) = limiter(leftPerB->at(fsgrids::bfield::PERBZ),centPerB->at(fsgrids::bfield::PERBZ),rghtPerB->at(fsgrids::bfield::PERBZ)); + + // pres_e = const * np.power(rho_e, index) + dMoments->at(fsgrids::dmoments::dPedx) = Peconst * limiter(pow(leftMoments->at(fsgrids::moments::RHOQ)/physicalconstants::CHARGE,Parameters::electronPTindex),pow(centMoments->at(fsgrids::moments::RHOQ)/physicalconstants::CHARGE,Parameters::electronPTindex),pow(rghtMoments->at(fsgrids::moments::RHOQ)/physicalconstants::CHARGE,Parameters::electronPTindex)); + if (Parameters::ohmHallTerm < 2 || sysBoundaryLayer == 1) { dPerB->at(fsgrids::dperb::dPERBydxx) = 0.0; dPerB->at(fsgrids::dperb::dPERBzdxx) = 0.0; @@ -147,6 +156,9 @@ void calculateDerivatives( dPerB->at(fsgrids::dperb::dPERBxdy) = limiter(leftPerB->at(fsgrids::bfield::PERBX),centPerB->at(fsgrids::bfield::PERBX),rghtPerB->at(fsgrids::bfield::PERBX)); dPerB->at(fsgrids::dperb::dPERBzdy) = limiter(leftPerB->at(fsgrids::bfield::PERBZ),centPerB->at(fsgrids::bfield::PERBZ),rghtPerB->at(fsgrids::bfield::PERBZ)); + // pres_e = const * np.power(rho_e, index) + dMoments->at(fsgrids::dmoments::dPedy) = Peconst * limiter(pow(leftMoments->at(fsgrids::moments::RHOQ)/physicalconstants::CHARGE,Parameters::electronPTindex),pow(centMoments->at(fsgrids::moments::RHOQ)/physicalconstants::CHARGE,Parameters::electronPTindex),pow(rghtMoments->at(fsgrids::moments::RHOQ)/physicalconstants::CHARGE,Parameters::electronPTindex)); + if (Parameters::ohmHallTerm < 2 || sysBoundaryLayer == 1) { dPerB->at(fsgrids::dperb::dPERBxdyy) = 0.0; dPerB->at(fsgrids::dperb::dPERBzdyy) = 0.0; @@ -183,6 +195,10 @@ void calculateDerivatives( dPerB->at(fsgrids::dperb::dPERBxdz) = limiter(leftPerB->at(fsgrids::bfield::PERBX),centPerB->at(fsgrids::bfield::PERBX),rghtPerB->at(fsgrids::bfield::PERBX)); dPerB->at(fsgrids::dperb::dPERBydz) = limiter(leftPerB->at(fsgrids::bfield::PERBY),centPerB->at(fsgrids::bfield::PERBY),rghtPerB->at(fsgrids::bfield::PERBY)); + + // pres_e = const * np.power(rho_e, index) + dMoments->at(fsgrids::dmoments::dPedz) = Peconst * limiter(pow(leftMoments->at(fsgrids::moments::RHOQ)/physicalconstants::CHARGE,Parameters::electronPTindex),pow(centMoments->at(fsgrids::moments::RHOQ)/physicalconstants::CHARGE,Parameters::electronPTindex),pow(rghtMoments->at(fsgrids::moments::RHOQ)/physicalconstants::CHARGE,Parameters::electronPTindex)); + if (Parameters::ohmHallTerm < 2 || sysBoundaryLayer == 1) { dPerB->at(fsgrids::dperb::dPERBxdzz) = 0.0; dPerB->at(fsgrids::dperb::dPERBydzz) = 0.0; @@ -286,13 +302,13 @@ void calculateDerivatives( * \sa calculateDerivatives calculateBVOLDerivativesSimple calculateBVOLDerivatives */ void calculateDerivativesSimple( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, cint& RKCase, const bool communicateMoments) { @@ -378,8 +394,8 @@ void calculateDerivativesSimple( */ void calculateBVOLDerivatives( - FsGrid< std::array, 2> & volGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, @@ -448,8 +464,8 @@ void calculateBVOLDerivatives( * \sa calculateDerivatives calculateBVOLDerivatives calculateDerivativesSimple */ void calculateBVOLDerivativesSimple( - FsGrid< std::array, 2> & volGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries ) { int timer; diff --git a/fieldsolver/derivatives.hpp b/fieldsolver/derivatives.hpp index 32c8bac52..98e15239b 100644 --- a/fieldsolver/derivatives.hpp +++ b/fieldsolver/derivatives.hpp @@ -22,27 +22,28 @@ #include +#include "../definitions.h" #include "../spatial_cell.hpp" #include "../sysboundary/sysboundary.h" #include "fs_limiters.h" void calculateDerivativesSimple( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, cint& RKCase, const bool communicateMoments); void calculateBVOLDerivativesSimple( - FsGrid< std::array, 2> & volGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries ); diff --git a/fieldsolver/fs_common.cpp b/fieldsolver/fs_common.cpp index 6db4507c8..3bb5dc022 100644 --- a/fieldsolver/fs_common.cpp +++ b/fieldsolver/fs_common.cpp @@ -52,8 +52,8 @@ Real divideIfNonZero( * \param reconstructionOrder Reconstruction order of the fields after Balsara 2009, 2 used for BVOL, 3 used for 2nd-order Hall term calculations. */ void reconstructionCoefficients( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, Real* perturbedResult, cint i, cint j, @@ -67,7 +67,7 @@ void reconstructionCoefficients( std::array * cep_i1j2k1 = NULL; std::array * cep_i1j1k2 = NULL; - FsGrid< std::array, 2> * params = & perBGrid; + FsGrid< std::array, FS_STENCIL_WIDTH> * params = & perBGrid; cep_i1j1k1 = params->get(i,j,k); dummyCellParams = cep_i1j1k1; diff --git a/fieldsolver/fs_common.h b/fieldsolver/fs_common.h index be37fe226..d637b7d1b 100644 --- a/fieldsolver/fs_common.h +++ b/fieldsolver/fs_common.h @@ -35,6 +35,7 @@ #include #include +#include "../definitions.h" #include "../common.h" #include "../parameters.h" #include "../projects/project.h" @@ -62,19 +63,19 @@ static creal EPS = 1.0e-30; using namespace std; bool initializeFieldPropagator( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< std::array, 2> & volGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries ); @@ -83,19 +84,19 @@ bool initializeFieldPropagatorAfterRebalance(); bool finalizeFieldPropagator(); bool propagateFields( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< std::array, 2> & volGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, creal& dt, cuint subcycles @@ -115,8 +116,8 @@ namespace Rec { } void reconstructionCoefficients( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, Real* perturbedResult, cint i, cint j, diff --git a/fieldsolver/gridGlue.cpp b/fieldsolver/gridGlue.cpp index 585cfa87a..8f47db25a 100644 --- a/fieldsolver/gridGlue.cpp +++ b/fieldsolver/gridGlue.cpp @@ -6,6 +6,12 @@ #include "../common.h" #include "gridGlue.hpp" + + + + + + /* Calculate the number of cells on the maximum refinement level overlapping the list of dccrg cells in cells. */ @@ -58,16 +64,17 @@ template void computeCoupling(dccrg::Dccrg globalIndices = momentsGrid.getGlobalIndices(i,j,k); - const dccrg::Types<3>::indices_t indices = {{(uint64_t)globalIndices[0], - (uint64_t)globalIndices[1], - (uint64_t)globalIndices[2]}}; //cast to avoid warnings - CellID dccrgCell = mpiGrid.get_existing_cell(indices, 0, mpiGrid.mapping.get_maximum_refinement_level()); - int process = mpiGrid.get_process(dccrgCell); - int64_t fsgridLid = momentsGrid.LocalIDForCoords(i,j,k); - int64_t fsgridGid = momentsGrid.GlobalIDForCoords(i,j,k); - onFsgridMapRemoteProcess[process].insert(dccrgCell); //cells are ordered (sorted) in set - onFsgridMapCells[dccrgCell].push_back(fsgridLid); + const std::array globalIndices = momentsGrid.getGlobalIndices(i,j,k); + const dccrg::Types<3>::indices_t indices = {{(uint64_t)globalIndices[0], + (uint64_t)globalIndices[1], + (uint64_t)globalIndices[2]}}; //cast to avoid warnings + CellID dccrgCell = mpiGrid.get_existing_cell(indices, 0, mpiGrid.mapping.get_maximum_refinement_level()); + + int process = mpiGrid.get_process(dccrgCell); + int64_t fsgridLid = momentsGrid.LocalIDForCoords(i,j,k); + int64_t fsgridGid = momentsGrid.GlobalIDForCoords(i,j,k); + onFsgridMapRemoteProcess[process].insert(dccrgCell); //cells are ordered (sorted) in set + onFsgridMapCells[dccrgCell].push_back(fsgridLid); } } } @@ -87,7 +94,10 @@ template void computeCoupling(dccrg::Dccrg& mpiGrid, const std::vector& cells, - FsGrid< std::array, 2>& momentsGrid, bool dt2 /*=false*/) { + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, + + bool dt2 /*=false*/) { int ii; //sorted list of dccrg cells. cells is typicall already sorted, but just to make sure.... @@ -179,13 +189,124 @@ void feedMomentsIntoFsGrid(dccrg::Dccrg& MPI_Waitall(sendRequests.size(), sendRequests.data(), MPI_STATUSES_IGNORE); + + + if (P::amrMaxSpatialRefLevel>0) { + + /*----------------------Filtering------------------------*/ + phiprof::start("BoxCar Filtering"); + + // Kernel Characteristics + // int stencilWidth = sizeof( kernel) / sizeof( kernel[0]); //get the stnecil width + // int kernelOffset= stencilWidth/3; //this is the kernel offset -int + int stencilWidth = 3; // dimension size of a 3D kernel + int kernelOffset = 1; // offset of 3 point stencil 3D kernel + Real kernelWeight= 1.0/27.0; // 3D boxcar kernel weight + + + // Update momentsGrid Ghost Cells + phiprof::start("GhostUpdate"); + momentsGrid.updateGhostCells(); //update ghost zones + phiprof::stop("GhostUpdate"); + + // Get size of local domain and create swapGrid for filtering + const int *mntDims= &momentsGrid.getLocalSize()[0]; //get local size for each proc. + FsGrid, FS_STENCIL_WIDTH> swapGrid = momentsGrid; //swap array + const int maxRefLevel = mpiGrid.mapping.get_maximum_refinement_level(); + + // Filtering Loop + std::vector::iterator maxNumPassesPtr; + int maxNumPasses; + maxNumPassesPtr=std::max_element(P::numPasses.begin(), P::numPasses.end()); + if(maxNumPassesPtr != P::numPasses.end()){ + maxNumPasses = *maxNumPassesPtr; + }else{ + std::cerr << "Trying to dereference null pointer \t" << " in " << __FILE__ << ":" << __LINE__ << std::endl; + abort(); + } + + for (int blurPass = 0; blurPass < maxNumPasses; blurPass++){ + + // Blurring Pass + phiprof::start("BlurPass"); + #pragma omp parallel for collapse(2) + for (int kk = 0; kk < mntDims[2]; kk++){ + for (int jj = 0; jj < mntDims[1]; jj++){ + for (int ii = 0; ii < mntDims[0]; ii++){ + + // Get refLevel level + int refLevel = technicalGrid.get(ii, jj, kk)->refLevel; + + // Skip pass + if (blurPass >= P::numPasses.at(refLevel) || + technicalGrid.get(ii, jj, kk)->sysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE || + (technicalGrid.get(ii, jj, kk)->sysBoundaryFlag != sysboundarytype::NOT_SYSBOUNDARY && technicalGrid.get(ii, jj, kk)->sysBoundaryLayer == 2) + ) + { + + continue; + } + + // Define some Pointers + std::array *cell; + std::array *swap; + + // Set Cell to zero before passing filter + swap = swapGrid.get(ii,jj,kk); + for (int e = 0; e < fsgrids::moments::N_MOMENTS; ++e) { + + swap->at(e)=0.0; + + } + + // Perform the blur + for (int a=0; aat(e)+=cell->at(e) * kernelWeight; + + } + } + } + } + } + } + } + + phiprof::stop("BlurPass"); + + + // Copy swapGrid back to momentsGrid + momentsGrid=swapGrid; + + // Update Ghost Cells + phiprof::start("GhostUpdate"); + momentsGrid.updateGhostCells(); + phiprof::stop("GhostUpdate"); + + } + + + phiprof::stop("BoxCar Filtering"); + + } } + void getFieldsFromFsGrid( - FsGrid< std::array, 2>& volumeFieldsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volumeFieldsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, dccrg::Dccrg& mpiGrid, const std::vector& cells ) { @@ -367,7 +488,6 @@ void getFieldsFromFsGrid( } MPI_Waitall(sendRequests.size(), sendRequests.data(), MPI_STATUSES_IGNORE); - } /* diff --git a/fieldsolver/gridGlue.hpp b/fieldsolver/gridGlue.hpp index 0b085d2e5..589070aff 100644 --- a/fieldsolver/gridGlue.hpp +++ b/fieldsolver/gridGlue.hpp @@ -1,4 +1,5 @@ #pragma once +#include "../definitions.h" #include #include #include @@ -16,7 +17,8 @@ std::vector mapDccrgIdToFsGridGlobalID(dccrg::Dccrg& mpiGrid, const std::vector& cells, - FsGrid< std::array, 2>& momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, bool dt2=false); /*! Copy field solver result (VOLB, VOLE, VOLPERB derivatives, gradpe) and store them back into DCCRG @@ -26,10 +28,10 @@ void feedMomentsIntoFsGrid(dccrg::Dccrg& * * This function assumes that proper grid coupling has been set up. */ -void getFieldsFromFsGrid(FsGrid< std::array, 2>& volumeFieldsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, +void getFieldsFromFsGrid(FsGrid< std::array, FS_STENCIL_WIDTH> & volumeFieldsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, dccrg::Dccrg& mpiGrid, const std::vector& cells ); @@ -42,8 +44,8 @@ void getFieldsFromFsGrid(FsGrid< std::array, 2> * This function assumes that proper grid coupling has been set up. */ void getBgFieldsAndDerivativesFromFsGrid( - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, dccrg::Dccrg& mpiGrid, const std::vector& cells ); @@ -53,9 +55,9 @@ void getBgFieldsAndDerivativesFromFsGrid( * This should only be neccessary for debugging. */ void getDerivativesFromFsGrid( - FsGrid< std::array, 2>& dperbGrid, - FsGrid< std::array, 2>& dmomentsGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dperbGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dmomentsGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, dccrg::Dccrg& mpiGrid, const std::vector& cells ); @@ -77,8 +79,8 @@ int getNumberOfCellsOnMaxRefLvl(dccrg::Dccrg void getFieldDataFromFsGrid( - FsGrid< std::array, 2>& sourceGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & sourceGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, dccrg::Dccrg& mpiGrid, const std::vector& cells, int index diff --git a/fieldsolver/ldz_electric_field.cpp b/fieldsolver/ldz_electric_field.cpp index e96337ddb..d03a8a38b 100644 --- a/fieldsolver/ldz_electric_field.cpp +++ b/fieldsolver/ldz_electric_field.cpp @@ -93,11 +93,11 @@ Real calculateCflSpeed( * \param ret_vW Whistler speed returned */ void calculateWaveSpeedYZ( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, cint i, cint j, cint k, @@ -207,11 +207,11 @@ void calculateWaveSpeedYZ( * \param ret_vW Whistler speed returned */ void calculateWaveSpeedXZ( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, cint i, cint j, cint k, @@ -321,11 +321,11 @@ void calculateWaveSpeedXZ( * \param ret_vW Whistler speed returned */ void calculateWaveSpeedXY( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, cint i, cint j, cint k, @@ -425,15 +425,15 @@ void calculateWaveSpeedXY( * \param RKCase Element in the enum defining the Runge-Kutta method steps */ void calculateEdgeElectricFieldX( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, @@ -783,15 +783,15 @@ void calculateEdgeElectricFieldX( * \param RKCase Element in the enum defining the Runge-Kutta method steps */ void calculateEdgeElectricFieldY( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, @@ -1140,15 +1140,15 @@ void calculateEdgeElectricFieldY( * \param RKCase Element in the enum defining the Runge-Kutta method steps */ void calculateEdgeElectricFieldZ( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, @@ -1512,15 +1512,15 @@ void calculateEdgeElectricFieldZ( * */ void calculateElectricField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, @@ -1616,18 +1616,18 @@ void calculateElectricField( * \sa calculateElectricField calculateEdgeElectricFieldX calculateEdgeElectricFieldY calculateEdgeElectricFieldZ */ void calculateUpwindedElectricFieldSimple( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, cint& RKCase ) { diff --git a/fieldsolver/ldz_electric_field.hpp b/fieldsolver/ldz_electric_field.hpp index ece919e7d..f286c5965 100644 --- a/fieldsolver/ldz_electric_field.hpp +++ b/fieldsolver/ldz_electric_field.hpp @@ -23,18 +23,18 @@ #include "fs_common.h" void calculateUpwindedElectricFieldSimple( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, cint& RKCase ); diff --git a/fieldsolver/ldz_gradpe.cpp b/fieldsolver/ldz_gradpe.cpp index dd965f612..365b00b60 100644 --- a/fieldsolver/ldz_gradpe.cpp +++ b/fieldsolver/ldz_gradpe.cpp @@ -30,9 +30,9 @@ using namespace std; void calculateEdgeGradPeTermXComponents( - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k @@ -47,8 +47,9 @@ void calculateEdgeGradPeTermXComponents( case 1: rhoq = momentsGrid.get(i,j,k)->at(fsgrids::moments::RHOQ); hallRhoq = (rhoq <= Parameters::hallMinimumRhoq ) ? Parameters::hallMinimumRhoq : rhoq ; - EGradPeGrid.get(i,j,k)->at(fsgrids::egradpe::EXGRADPE) = -physicalconstants::K_B*Parameters::electronTemperature*dMomentsGrid.get(i,j,k)->at(fsgrids::dmoments::drhoqdx) / (hallRhoq*EGradPeGrid.DX); - break; + //EGradPeGrid.get(i,j,k)->at(fsgrids::egradpe::EXGRADPE) = -physicalconstants::K_B*Parameters::electronTemperature*dMomentsGrid.get(i,j,k)->at(fsgrids::dmoments::drhoqdx) / (hallRhoq*EGradPeGrid.DX); + EGradPeGrid.get(i,j,k)->at(fsgrids::egradpe::EXGRADPE) = - dMomentsGrid.get(i,j,k)->at(fsgrids::dmoments::dPedx) / (hallRhoq*EGradPeGrid.DX); + break; default: cerr << __FILE__ << ":" << __LINE__ << "You are welcome to code higher-order Hall term correction terms." << endl; @@ -57,9 +58,9 @@ void calculateEdgeGradPeTermXComponents( } void calculateEdgeGradPeTermYComponents( - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k @@ -74,7 +75,8 @@ void calculateEdgeGradPeTermYComponents( case 1: rhoq = momentsGrid.get(i,j,k)->at(fsgrids::moments::RHOQ); hallRhoq = (rhoq <= Parameters::hallMinimumRhoq ) ? Parameters::hallMinimumRhoq : rhoq ; - EGradPeGrid.get(i,j,k)->at(fsgrids::egradpe::EYGRADPE) = -physicalconstants::K_B*Parameters::electronTemperature*dMomentsGrid.get(i,j,k)->at(fsgrids::dmoments::drhoqdy) / (hallRhoq*EGradPeGrid.DY); + //EGradPeGrid.get(i,j,k)->at(fsgrids::egradpe::EYGRADPE) = -physicalconstants::K_B*Parameters::electronTemperature*dMomentsGrid.get(i,j,k)->at(fsgrids::dmoments::drhoqdy) / (hallRhoq*EGradPeGrid.DY); + EGradPeGrid.get(i,j,k)->at(fsgrids::egradpe::EYGRADPE) = - dMomentsGrid.get(i,j,k)->at(fsgrids::dmoments::dPedy) / (hallRhoq*EGradPeGrid.DY); break; default: @@ -84,9 +86,9 @@ void calculateEdgeGradPeTermYComponents( } void calculateEdgeGradPeTermZComponents( - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k @@ -101,7 +103,8 @@ void calculateEdgeGradPeTermZComponents( case 1: rhoq = momentsGrid.get(i,j,k)->at(fsgrids::moments::RHOQ); hallRhoq = (rhoq <= Parameters::hallMinimumRhoq ) ? Parameters::hallMinimumRhoq : rhoq ; - EGradPeGrid.get(i,j,k)->at(fsgrids::egradpe::EZGRADPE) = -physicalconstants::K_B*Parameters::electronTemperature*dMomentsGrid.get(i,j,k)->at(fsgrids::dmoments::drhoqdz) / (hallRhoq*EGradPeGrid.DZ); + //EGradPeGrid.get(i,j,k)->at(fsgrids::egradpe::EZGRADPE) = -physicalconstants::K_B*Parameters::electronTemperature*dMomentsGrid.get(i,j,k)->at(fsgrids::dmoments::drhoqdz) / (hallRhoq*EGradPeGrid.DZ); + EGradPeGrid.get(i,j,k)->at(fsgrids::egradpe::EZGRADPE) = - dMomentsGrid.get(i,j,k)->at(fsgrids::dmoments::dPedz) / (hallRhoq*EGradPeGrid.DZ); break; default: @@ -114,10 +117,10 @@ void calculateEdgeGradPeTermZComponents( * @param sysBoundaries System boundary condition functions. */ void calculateGradPeTerm( - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, @@ -148,11 +151,11 @@ void calculateGradPeTerm( } void calculateGradPeTermSimple( - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, cint& RKCase ) { diff --git a/fieldsolver/ldz_gradpe.hpp b/fieldsolver/ldz_gradpe.hpp index 827f346f2..23caaf2f1 100644 --- a/fieldsolver/ldz_gradpe.hpp +++ b/fieldsolver/ldz_gradpe.hpp @@ -20,12 +20,14 @@ #ifndef LDZ_GRADPE_HPP #define LDZ_GRADPE_HPP +#include "../definitions.h" + void calculateGradPeTermSimple( - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, cint& RKCase ); diff --git a/fieldsolver/ldz_hall.cpp b/fieldsolver/ldz_hall.cpp index 9f20c20a1..066bacd94 100644 --- a/fieldsolver/ldz_hall.cpp +++ b/fieldsolver/ldz_hall.cpp @@ -433,13 +433,13 @@ REAL JXBZ_110_111( * */ void calculateEdgeHallTermXComponents( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, const Real* const perturbedCoefficients, cint i, cint j, @@ -531,13 +531,13 @@ void calculateEdgeHallTermXComponents( * */ void calculateEdgeHallTermYComponents( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, const Real* const perturbedCoefficients, cint i, cint j, @@ -629,13 +629,13 @@ void calculateEdgeHallTermYComponents( * */ void calculateEdgeHallTermZComponents( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, const Real* const perturbedCoefficients, cint i, cint j, @@ -724,13 +724,13 @@ void calculateEdgeHallTermZComponents( * \sa calculateHallTermSimple calculateEdgeHallTermXComponents calculateEdgeHallTermYComponents calculateEdgeHallTermZComponents */ void calculateHallTerm( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, cint i, cint j, @@ -794,15 +794,15 @@ void calculateHallTerm( * \sa calculateHallTerm */ void calculateHallTermSimple( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, cint& RKCase, const bool communicateMomentsDerivatives diff --git a/fieldsolver/ldz_hall.hpp b/fieldsolver/ldz_hall.hpp index a7c1e7a6a..12c1af40c 100644 --- a/fieldsolver/ldz_hall.hpp +++ b/fieldsolver/ldz_hall.hpp @@ -20,16 +20,18 @@ #ifndef LDZ_HALL_HPP #define LDZ_HALL_HPP +#include "../definitions.h" + void calculateHallTermSimple( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, cint& RKCase, const bool communicateMomentsDerivatives diff --git a/fieldsolver/ldz_magnetic_field.cpp b/fieldsolver/ldz_magnetic_field.cpp index 36ccf6d1a..492aafbfa 100644 --- a/fieldsolver/ldz_magnetic_field.cpp +++ b/fieldsolver/ldz_magnetic_field.cpp @@ -47,10 +47,10 @@ * \param doZ If true, compute the z component (default true). */ void propagateMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EDt2Grid, cint i, cint j, cint k, @@ -172,11 +172,11 @@ void propagateMagneticField( * \sa propagateMagneticFieldSimple propagateMagneticField */ void propagateSysBoundaryMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EDt2Grid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, @@ -185,13 +185,11 @@ void propagateSysBoundaryMagneticField( cint& RKCase, cuint component ) { - std::array * bGrid; if (RKCase == RK_ORDER1 || RKCase == RK_ORDER2_STEP2) { - bGrid = perBGrid.get(i,j,k); + perBGrid.get(i,j,k)->at(fsgrids::bfield::PERBX + component) = sysBoundaries.getSysBoundary(technicalGrid.get(i,j,k)->sysBoundaryFlag)->fieldSolverBoundaryCondMagneticField(perBGrid, technicalGrid, i, j, k, dt, component); } else { - bGrid = perBDt2Grid.get(i,j,k); + perBDt2Grid.get(i,j,k)->at(fsgrids::bfield::PERBX + component) = sysBoundaries.getSysBoundary(technicalGrid.get(i,j,k)->sysBoundaryFlag)->fieldSolverBoundaryCondMagneticField(perBDt2Grid, technicalGrid, i, j, k, dt, component); } - bGrid->at(fsgrids::bfield::PERBX + component) = sysBoundaries.getSysBoundary(technicalGrid.get(i,j,k)->sysBoundaryFlag)->fieldSolverBoundaryCondMagneticField(perBGrid, perBDt2Grid, EGrid, EDt2Grid, technicalGrid, i, j, k, dt, RKCase, component); } /*! \brief High-level magnetic field propagation function. @@ -210,11 +208,11 @@ void propagateSysBoundaryMagneticField( * \sa propagateMagneticField propagateSysBoundaryMagneticField */ void propagateMagneticFieldSimple( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EDt2Grid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, creal& dt, cint& RKCase @@ -281,7 +279,21 @@ void propagateMagneticFieldSimple( } } } + phiprof::stop(timer); + timer=phiprof::initializeTimer("MPI","MPI"); + phiprof::start(timer); + if (RKCase == RK_ORDER1 || RKCase == RK_ORDER2_STEP2) { + // Exchange PERBX,PERBY,PERBZ with neighbours + perBGrid.updateGhostCells(); + } else { // RKCase == RK_ORDER2_STEP1 + // Exchange PERBX_DT2,PERBY_DT2,PERBZ_DT2 with neighbours + perBDt2Grid.updateGhostCells(); + } + phiprof::stop(timer); + + timer=phiprof::initializeTimer("Compute system boundary cells"); + phiprof::start(timer); // L2 pass #pragma omp parallel for collapse(3) for (int k=0; k, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EDt2Grid, cint i, cint j, cint k, @@ -44,11 +44,11 @@ void propagateMagneticField( ); void propagateMagneticFieldSimple( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EDt2Grid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, creal& dt, cint& RKCase diff --git a/fieldsolver/ldz_main.cpp b/fieldsolver/ldz_main.cpp index 2894cfb64..6bf5d334e 100644 --- a/fieldsolver/ldz_main.cpp +++ b/fieldsolver/ldz_main.cpp @@ -84,19 +84,19 @@ bool finalizeFieldPropagator() { * */ bool propagateFields( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< std::array, 2> & EHallGrid, - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, - FsGrid< std::array, 2> & BgBGrid, - FsGrid< std::array, 2> & volGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, creal& dt, cuint subcycles diff --git a/fieldsolver/ldz_volume.cpp b/fieldsolver/ldz_volume.cpp index e406d03b0..3256777b1 100644 --- a/fieldsolver/ldz_volume.cpp +++ b/fieldsolver/ldz_volume.cpp @@ -32,11 +32,11 @@ using namespace std; void calculateVolumeAveragedFields( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & volGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { //const std::array gridDims = technicalGrid.getLocalSize(); const int* gridDims = &technicalGrid.getLocalSize()[0]; diff --git a/fieldsolver/ldz_volume.hpp b/fieldsolver/ldz_volume.hpp index 8277822f0..4c86915d8 100644 --- a/fieldsolver/ldz_volume.hpp +++ b/fieldsolver/ldz_volume.hpp @@ -32,11 +32,11 @@ * \sa reconstructionCoefficients */ void calculateVolumeAveragedFields( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & volGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); #endif diff --git a/grid.cpp b/grid.cpp index 0bd98831a..ff0bfe5ec 100644 --- a/grid.cpp +++ b/grid.cpp @@ -56,6 +56,8 @@ using namespace std; using namespace phiprof; +int globalflags::AMRstencilWidth = VLASOV_STENCIL_WIDTH; + extern Logger logFile, diagnostic; void initVelocityGridGeometry(dccrg::Dccrg& mpiGrid); @@ -85,14 +87,14 @@ void initializeGrids( int argn, char **argc, dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, Project& project ) { @@ -109,7 +111,26 @@ void initializeGrids( } MPI_Comm comm = MPI_COMM_WORLD; - int neighborhood_size = max(FS_STENCIL_WIDTH, VLASOV_STENCIL_WIDTH); + int neighborhood_size = VLASOV_STENCIL_WIDTH; + if (P::amrMaxSpatialRefLevel > 0) { + switch (VLASOV_STENCIL_WIDTH) { + case 1: + // Required cells will be included already + break; + case 2: + // looking from high to low refinement: stencil 2 will only give 1 cell, so need to add 1 + neighborhood_size = VLASOV_STENCIL_WIDTH+1; + break; + case 3: + // looking from high to low refinement: stencil 3 will only give 2 cells, so need to add 2 + // to reach surely into the third low-refinement neighbour + neighborhood_size = VLASOV_STENCIL_WIDTH+2; + break; + default: + std::cerr<<"Warning: unrecognized VLASOV_STENCIL_WIDTH in grid.cpp"< grid_length = {{P::xcells_ini, P::ycells_ini, P::zcells_ini}}; dccrg::Cartesian_Geometry::Parameters geom_params; @@ -175,6 +196,7 @@ void initializeGrids( } phiprof::stop("Classify cells (sys boundary conditions)"); + // Check refined cells do not touch boundary cells phiprof::start("Check boundary refinement"); if(!sysBoundaries.checkRefinement(mpiGrid)) { @@ -200,7 +222,32 @@ void initializeGrids( } phiprof::stop("Apply system boundary conditions state"); } - + + if (P::amrMaxSpatialRefLevel>0) { + // Map Refinement Level to FsGrid + phiprof::start("Map Refinement Level to FsGrid"); + const int *localDims = &momentsGrid.getLocalSize()[0]; + + // #pragma omp parallel for collapse(3) + for (int k=0; k mapIndices = momentsGrid.getGlobalIndices(i,j,k); + const dccrg::Types<3>::indices_t indices = {{(uint64_t)mapIndices[0],(uint64_t)mapIndices[1],(uint64_t)mapIndices[2]}}; //cast to avoid warnings + CellID dccrgCellID2 = mpiGrid.get_existing_cell(indices, 0, mpiGrid.mapping.get_maximum_refinement_level()); + int amrLevel= mpiGrid.get_refinement_level(dccrgCellID2); + technicalGrid.get(i, j, k)-> refLevel =amrLevel ; + } + } + } + phiprof::stop("Map Refinement Level to FsGrid"); + } + + + // Update technicalGrid + technicalGrid.updateGhostCells(); // This needs to be done at some point + if (!P::isRestart) { //Initial state based on project, background field in all cells //and other initial values in non-sysboundary cells @@ -260,9 +307,11 @@ void initializeGrids( phiprof::start("Init moments"); calculateInitialVelocityMoments(mpiGrid); phiprof::stop("Init moments"); - */ + */ + } - + + // Init mesh data container if (getObjectWrapper().meshData.initialize("SpatialGrid") == false) { cerr << "(Grid) Failed to initialize mesh data container in " << __FILE__ << ":" << __LINE__ << endl; @@ -304,16 +353,15 @@ void initializeGrids( phiprof::stop("setProjectBField"); phiprof::start("Finish fsgrid setup"); - feedMomentsIntoFsGrid(mpiGrid, cells, momentsGrid,false); + feedMomentsIntoFsGrid(mpiGrid, cells, momentsGrid,technicalGrid, false); if(!P::isRestart) { // WARNING this means moments and dt2 moments are the same here at t=0, which is a feature so far. - feedMomentsIntoFsGrid(mpiGrid, cells, momentsDt2Grid,false); + feedMomentsIntoFsGrid(mpiGrid, cells, momentsDt2Grid, technicalGrid, false); } else { - feedMomentsIntoFsGrid(mpiGrid, cells, momentsDt2Grid,true); + feedMomentsIntoFsGrid(mpiGrid, cells, momentsDt2Grid, technicalGrid, true); } momentsGrid.updateGhostCells(); momentsDt2Grid.updateGhostCells(); - technicalGrid.updateGhostCells(); // This needs to be done at some point phiprof::stop("Finish fsgrid setup"); phiprof::stop("Set initial state"); @@ -599,6 +647,7 @@ bool adjustVelocityBlocks(dccrg::Dccrg& m // gather spatial neighbor list and create vector with pointers to neighbor spatial cells const auto* neighbors = mpiGrid.get_neighbors_of(cell_id, NEAREST_NEIGHBORHOOD_ID); + // Note: at AMR refinement boundaries this can cause blocks to propagate further than absolutely required vector neighbor_ptrs; neighbor_ptrs.reserve(neighbors->size()); @@ -732,8 +781,11 @@ void deallocateRemoteCellBlocks(dccrg::Dccrg& mpiGrid, - const uint popID) +void updateRemoteVelocityBlockLists( + dccrg::Dccrg& mpiGrid, + const uint popID, + const uint neighborhood/*=DIST_FUNC_NEIGHBORHOOD_ID default*/ +) { SpatialCell::setCommunicatedSpecies(popID); @@ -743,22 +795,23 @@ void updateRemoteVelocityBlockLists(dccrg::Dccrg incoming_cells - = mpiGrid.get_remote_cells_on_process_boundary(DIST_FUNC_NEIGHBORHOOD_ID); + = mpiGrid.get_remote_cells_on_process_boundary(neighborhood); #pragma omp parallel for for (unsigned int i=0; i& mpi // set reduced neighborhoods typedef dccrg::Types<3>::neighborhood_item_t neigh_t; - // set a reduced neighborhood for field solver + // set a reduced neighborhood for nearest neighbours std::vector neighborhood; for (int z = -1; z <= 1; z++) { for (int y = -1; y <= 1; y++) { @@ -861,7 +915,7 @@ void initializeStencils(dccrg::Dccrg& mpi } } } - mpiGrid.add_neighborhood(FIELD_SOLVER_NEIGHBORHOOD_ID, neighborhood); + //mpiGrid.add_neighborhood(FIELD_SOLVER_NEIGHBORHOOD_ID, neighborhood); mpiGrid.add_neighborhood(NEAREST_NEIGHBORHOOD_ID, neighborhood); mpiGrid.add_neighborhood(SYSBOUNDARIES_NEIGHBORHOOD_ID, neighborhood); @@ -879,8 +933,43 @@ void initializeStencils(dccrg::Dccrg& mpi } mpiGrid.add_neighborhood(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID, neighborhood); - /*add face neighbors if stencil width larger than 2*/ - for (int d = 3; d <= VLASOV_STENCIL_WIDTH; d++) { + // In spatial AMR using DCCRG, the neighbors are considered relative to a given cell's size. + // To get two coarse neighbors from a fine cell at interfaces, the stencil size needs to be increased by one. + int addStencilDepth = 0; + if (P::amrMaxSpatialRefLevel > 0) { + switch (VLASOV_STENCIL_WIDTH) { + case 1: + // Required cells will be included already + break; + case 2: + // looking from high to low refinement: stencil 2 will only give 1 cell, so need to add 1 + addStencilDepth = 1; + break; + case 3: + // looking from high to low refinement: stencil 3 will only give 2 cells, so need to add 2 + // to reach surely into the third low-refinement neighbour + addStencilDepth = 2; + break; + default: + std::cerr<<"Warning: unrecognized VLASOV_STENCIL_WIDTH in grid.cpp"<& mpi neighborhood.push_back({{0, 0, d}}); neighborhood.push_back({{0, 0,-d}}); } - /*all possible communication pairs*/ mpiGrid.add_neighborhood(FULL_NEIGHBORHOOD_ID, neighborhood); - /*stencils for semilagrangian propagators*/ neighborhood.clear(); - for (int d = -VLASOV_STENCIL_WIDTH; d <= VLASOV_STENCIL_WIDTH; d++) { + for (int d = -VLASOV_STENCIL_WIDTH-addStencilDepth; d <= VLASOV_STENCIL_WIDTH+addStencilDepth; d++) { if (d != 0) { neighborhood.push_back({{d, 0, 0}}); neighborhood.push_back({{0, d, 0}}); @@ -921,16 +1008,16 @@ void initializeStencils(dccrg::Dccrg& mpi mpiGrid.add_neighborhood(DIST_FUNC_NEIGHBORHOOD_ID, neighborhood); neighborhood.clear(); - for (int d = -VLASOV_STENCIL_WIDTH; d <= VLASOV_STENCIL_WIDTH; d++) { + for (int d = -VLASOV_STENCIL_WIDTH-addStencilDepth; d <= VLASOV_STENCIL_WIDTH+addStencilDepth; d++) { if (d != 0) { neighborhood.push_back({{d, 0, 0}}); } } mpiGrid.add_neighborhood(VLASOV_SOLVER_X_NEIGHBORHOOD_ID, neighborhood); - + neighborhood.clear(); - for (int d = -VLASOV_STENCIL_WIDTH; d <= VLASOV_STENCIL_WIDTH; d++) { + for (int d = -VLASOV_STENCIL_WIDTH-addStencilDepth; d <= VLASOV_STENCIL_WIDTH+addStencilDepth; d++) { if (d != 0) { neighborhood.push_back({{0, d, 0}}); } @@ -939,7 +1026,7 @@ void initializeStencils(dccrg::Dccrg& mpi neighborhood.clear(); - for (int d = -VLASOV_STENCIL_WIDTH; d <= VLASOV_STENCIL_WIDTH; d++) { + for (int d = -VLASOV_STENCIL_WIDTH-addStencilDepth; d <= VLASOV_STENCIL_WIDTH+addStencilDepth; d++) { if (d != 0) { neighborhood.push_back({{0, 0, d}}); } @@ -1038,7 +1125,8 @@ bool validateMesh(dccrg::Dccrg& mpiGrid,c // Get all spatial neighbors //const vector* neighbors = mpiGrid.get_neighbors_of(cells[c],NEAREST_NEIGHBORHOOD_ID); const auto* neighbors = mpiGrid.get_neighbors_of(cells[c], NEAREST_NEIGHBORHOOD_ID); - + //#warning TODO should vAMR grandparents be checked only for face neighbors instead of NEAREST_NEIGHBORHOOD_ID? + // Iterate over all spatial neighbors // for (size_t n=0; nsize(); ++n) { diff --git a/grid.h b/grid.h index cd931ab14..8f170aa10 100644 --- a/grid.h +++ b/grid.h @@ -22,6 +22,7 @@ #ifndef GRID_H #define GRID_H +#include "definitions.h" #include "spatial_cell.hpp" #include #include @@ -36,14 +37,14 @@ void initializeGrids( int argn, char **argc, dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2> & momentsGrid, - FsGrid< std::array, 2> & momentsDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EGradPeGrid, - FsGrid< std::array, 2> & volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsDt2Grid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, SysBoundary& sysBoundaries, Project& project ); @@ -65,7 +66,8 @@ data. This is needed if one has locally adjusted velocity blocks */ void updateRemoteVelocityBlockLists( dccrg::Dccrg& mpiGrid, - const uint popID + const uint popID, + const uint neighborhood=DIST_FUNC_NEIGHBORHOOD_ID ); /*! Deallocates all blocks in remote cells in order to save diff --git a/ioread.cpp b/ioread.cpp index 7f99f8324..1fd704c66 100644 --- a/ioread.cpp +++ b/ioread.cpp @@ -778,13 +778,14 @@ bool readCellParamsVariable( } template bool readFsGridVariable( - vlsv::ParallelReader& file, const string& variableName, int numWritingRanks, FsGrid,2>& targetGrid) { + vlsv::ParallelReader& file, const string& variableName, int numWritingRanks, FsGrid,FS_STENCIL_WIDTH> & targetGrid) { uint64_t arraySize; uint64_t vectorSize; vlsv::datatype::type dataType; uint64_t byteSize; list > attribs; + bool convertFloatType = false; attribs.push_back(make_pair("name",variableName)); attribs.push_back(make_pair("mesh","fsgrid")); @@ -794,8 +795,8 @@ template bool readFsGridVariable( return false; } if(! (dataType == vlsv::datatype::type::FLOAT && byteSize == sizeof(Real))) { - logFile << "(RESTART) ERROR: Attempting to read fsgrid variable " << variableName << ", but it is not in the same floating point format as the simulation expects (" << byteSize*8 << " bits instead of " << sizeof(Real)*8 << ")." << endl << write; - return false; + logFile << "(RESTART) Converting floating point format of fsgrid variable " << variableName << " from " << byteSize * 8 << " bits to " << sizeof(Real) * 8 << " bits." << endl << write; + convertFloatType = true; } // Are we restarting from the same number of tasks, or a different number? @@ -829,9 +830,24 @@ template bool readFsGridVariable( // Read into buffer std::vector buffer(storageSize*N); - if(file.readArray("VARIABLE",attribs, localStartOffset, storageSize, (char*)buffer.data()) == false) { - logFile << "(RESTART) ERROR: Failed to read fsgrid variable " << variableName << endl << write; - return false; + if(!convertFloatType) { + if(file.readArray("VARIABLE",attribs, localStartOffset, storageSize, (char*)buffer.data()) == false) { + logFile << "(RESTART) ERROR: Failed to read fsgrid variable " << variableName << endl << write; + return false; + } + } else { + + // Read to temporary float buffer + std::vector readBuffer(storageSize*N); + + if(file.readArray("VARIABLE",attribs, localStartOffset, storageSize, (char*)readBuffer.data()) == false) { + logFile << "(RESTART) ERROR: Failed to read fsgrid variable " << variableName << endl << write; + return false; + } + + for(uint64_t i=0; i bool readFsGridVariable( // Read into buffer std::vector buffer(thatTasksSize[0]*thatTasksSize[1]*thatTasksSize[2]*N); - // TODO: Should these be multireads instead? And/or can this be parallelized? - if(file.readArray("VARIABLE",attribs, fileOffset, thatTasksSize[0]*thatTasksSize[1]*thatTasksSize[2], (char*)buffer.data()) == false) { - logFile << "(RESTART) ERROR: Failed to read fsgrid variable " << variableName << endl << write; - return false; - } + phiprof::start("readArray"); + file.startMultiread("VARIABLE", attribs); // Read every source rank that we have an overlap with. if(overlapSize[0]*overlapSize[1]*overlapSize[2] > 0) { + + if(!convertFloatType) { + if(file.addMultireadUnit((char*)buffer.data(), thatTasksSize[0]*thatTasksSize[1]*thatTasksSize[2])==false) { + logFile << "(RESTART) ERROR: Failed to read fsgrid variable " << variableName << endl << write; + return false; + } + file.endMultiread(fileOffset); + } else { + std::vector readBuffer(thatTasksSize[0]*thatTasksSize[1]*thatTasksSize[2]*N); + if(file.addMultireadUnit((char*)readBuffer.data(), thatTasksSize[0]*thatTasksSize[1]*thatTasksSize[2])==false) { + logFile << "(RESTART) ERROR: Failed to read fsgrid variable " << variableName << endl << write; + return false; + } + file.endMultiread(fileOffset); + + for(uint64_t i=0; i< thatTasksSize[0]*thatTasksSize[1]*thatTasksSize[2]*N; i++) { + buffer[i]=readBuffer[i]; + } + } + // Copy continuous stripes in x direction. for(int z=overlapStart[2]; z bool readFsGridVariable( } } } - } + } else { + // If we don't overlap, just perform a dummy read. + file.endMultiread(fileOffset); + } fileOffset += thatTasksSize[0] * thatTasksSize[1] * thatTasksSize[2]; + phiprof::stop("memcpy"); } } - + phiprof::start("updateGhostCells"); targetGrid.updateGhostCells(); + phiprof::stop("updateGhostCells"); return true; } @@ -981,9 +1019,9 @@ bool checkScalarParameter(vlsv::ParallelReader& file,const string& name,T correc \sa readGrid */ bool exec_readGrid(dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, const std::string& name) { vector fileCells; /*< CellIds for all cells in file*/ vector nBlocks;/*< Number of blocks for all cells in file*/ @@ -999,6 +1037,8 @@ bool exec_readGrid(dccrg::Dccrg& mpiGrid, phiprof::start("readGrid"); + phiprof::start("readScalars"); + vlsv::ParallelReader file; MPI_Info mpiInfo = MPI_INFO_NULL; @@ -1043,6 +1083,8 @@ bool exec_readGrid(dccrg::Dccrg& mpiGrid, checkScalarParameter(file,"ycells_ini",P::ycells_ini,MASTER_RANK,MPI_COMM_WORLD); checkScalarParameter(file,"zcells_ini",P::zcells_ini,MASTER_RANK,MPI_COMM_WORLD); + phiprof::stop("readScalars"); + phiprof::start("readDatalayout"); if (success == true) success = readCellIds(file,fileCells,MASTER_RANK,MPI_COMM_WORLD); @@ -1172,16 +1214,21 @@ bool exec_readGrid(dccrg::Dccrg& mpiGrid, } phiprof::stop("readBlockData"); + phiprof::start("updateMpiGridNeighbors"); mpiGrid.update_copies_of_remote_neighbors(FULL_NEIGHBORHOOD_ID); + phiprof::stop("updateMpiGridNeighbors"); + phiprof::start("readFsGrid"); // Read fsgrid data back in int fsgridInputRanks=0; if(readScalarParameter(file,"numWritingRanks",fsgridInputRanks, MASTER_RANK, MPI_COMM_WORLD) == false) { exitOnError(false, "(RESTART) FSGrid writing rank number not found in restart file", MPI_COMM_WORLD); } - success = readFsGridVariable(file, "fg_PERB", fsgridInputRanks, perBGrid); - success = readFsGridVariable(file, "fg_E", fsgridInputRanks, EGrid); + if(success) { success = readFsGridVariable(file, "fg_PERB", fsgridInputRanks, perBGrid); } + if(success) { success = readFsGridVariable(file, "fg_E", fsgridInputRanks, EGrid); } + exitOnError(success,"(RESTART) Failure reading fsgrid restart variables",MPI_COMM_WORLD); + phiprof::stop("readFsGrid"); success = file.close(); phiprof::stop("readGrid"); @@ -1197,9 +1244,9 @@ bool exec_readGrid(dccrg::Dccrg& mpiGrid, \param name Name of the restart file e.g. "restart.00052.vlsv" */ bool readGrid(dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, const std::string& name){ //Check the vlsv version from the file: return exec_readGrid(mpiGrid,perBGrid,EGrid,technicalGrid,name); diff --git a/ioread.h b/ioread.h index eabb28cd6..84fb3264a 100644 --- a/ioread.h +++ b/ioread.h @@ -26,6 +26,7 @@ #include #include +#include "definitions.h" #include "spatial_cell.hpp" #include "datareduction/datareducer.h" @@ -37,9 +38,9 @@ \param name Name of the restart file e.g. "restart.00052.vlsv" */ bool readGrid(dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, const std::string& name); diff --git a/iowrite.cpp b/iowrite.cpp index 07663b7cc..803c057ca 100644 --- a/iowrite.cpp +++ b/iowrite.cpp @@ -304,16 +304,16 @@ bool writeVelocityDistributionData(const uint popID,Writer& vlsvWriter, */ bool writeDataReducer(const dccrg::Dccrg& mpiGrid, const std::vector& cells, - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, const bool writeAsFloat, DataReducer& dataReducer, int dataReducerIndex, @@ -431,7 +431,7 @@ bool writeDataReducer(const dccrg::Dccrg& // If the data reducer didn't want to write dccrg data, maybe it will be happy // dumping data straight from fsgrid into our file. phiprof::start("writeFsGrid"); - success = dataReducer.writeFsGridData(perBGrid,EGrid,EHallGrid,EGradPeGrid,momentsGrid,dPerBGrid,dMomentsGrid,BgBGrid,volGrid, technicalGrid, "fsgrid", dataReducerIndex, vlsvWriter); + success = dataReducer.writeFsGridData(perBGrid,EGrid,EHallGrid,EGradPeGrid,momentsGrid,dPerBGrid,dMomentsGrid,BgBGrid,volGrid, technicalGrid, "fsgrid", dataReducerIndex, vlsvWriter, writeAsFloat); phiprof::stop("writeFsGrid"); } @@ -829,7 +829,7 @@ bool writeMeshBoundingBox( Writer & vlsvWriter, * @param technicalGrid An fsgrid instance used to extract metadata info. * @param vlsvWriter file object to write into. */ -bool writeFsGridMetadata(FsGrid< fsgrids::technical, 2>& technicalGrid, vlsv::Writer& vlsvWriter) { +bool writeFsGridMetadata(FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, vlsv::Writer& vlsvWriter) { std::map xmlAttributes; const std::string meshName="fsgrid"; @@ -901,7 +901,7 @@ bool writeFsGridMetadata(FsGrid< fsgrids::technical, 2>& technicalGrid, vlsv::Wr // writeDomainSizes - std::array meshDomainSize({globalIds.size(), 0}); + std::array meshDomainSize({globalIds.size(), 0}); vlsvWriter.writeArray("MESH_DOMAIN_SIZES", xmlAttributes, 1, 2, &meshDomainSize[0]); // how many MPI ranks we wrote from @@ -934,6 +934,10 @@ bool writeVelocitySpace(dccrg::Dccrg& mpi //Compute which cells will write out their velocity space vector velSpaceCells; int lineX, lineY, lineZ; + Real shellRadiusSquare; + Real cellX, cellY, cellZ, DX, DY, DZ; + Real dx_rm, dx_rp, dy_rm, dy_rp, dz_rm, dz_rp; + Real rsquare_minus,rsquare_plus; for (uint i = 0; i < cells.size(); i++) { mpiGrid[cells[i]]->parameters[CellParams::ISCELLSAVINGF] = 0.0; // CellID stride selection @@ -978,9 +982,37 @@ bool writeVelocitySpace(dccrg::Dccrg& mpi ) { velSpaceCells.push_back(cells[i]); mpiGrid[cells[i]]->parameters[CellParams::ISCELLSAVINGF] = 1.0; + continue; // Avoid double entries in case the cell also matches following conditions. } } } + + // Loop over spherical shells at defined distances + for (uint ishell = 0; ishell < P::systemWriteDistributionWriteShellRadius.size(); ishell++) { + shellRadiusSquare = P::systemWriteDistributionWriteShellRadius[ishell] * P::systemWriteDistributionWriteShellRadius[ishell]; + cellX = mpiGrid[cells[i]]->parameters[CellParams::XCRD]; + cellY = mpiGrid[cells[i]]->parameters[CellParams::YCRD]; + cellZ = mpiGrid[cells[i]]->parameters[CellParams::ZCRD]; + DX = mpiGrid[cells[i]]->parameters[CellParams::DX]; + DY = mpiGrid[cells[i]]->parameters[CellParams::DY]; + DZ = mpiGrid[cells[i]]->parameters[CellParams::DZ]; + + dx_rm = cellX < 0 ? DX : 0; + dx_rp = cellX < 0 ? 0 : DX; + dy_rm = cellY < 0 ? DY : 0; + dy_rp = cellY < 0 ? 0 : DY; + dz_rm = cellZ < 0 ? DZ : 0; + dz_rp = cellZ < 0 ? 0 : DZ; + rsquare_minus = (cellX + dx_rm) * (cellX + dx_rm) + (cellY + dy_rm) * (cellY + dy_rm) + (cellZ + dz_rm) * (cellZ + dz_rm); + rsquare_plus = (cellX + dx_rp) * (cellX + dx_rp) + (cellY + dy_rp) * (cellY + dy_rp) + (cellZ + dz_rp) * (cellZ + dz_rp); + if (rsquare_minus <= shellRadiusSquare && rsquare_plus > shellRadiusSquare && + P::systemWriteDistributionWriteShellStride[ishell] > 0 && + cells[i] % P::systemWriteDistributionWriteShellStride[ishell] == 0 + ) { + velSpaceCells.push_back(cells[i]); + mpiGrid[cells[i]]->parameters[CellParams::ISCELLSAVINGF] = 1.0; + } + } } uint64_t numVelSpaceCells; @@ -1030,19 +1062,20 @@ bool checkForSameMembers( const vector local_cells, const vector& mpiGrid, - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, DataReducer* dataReducer, const uint& index, - const bool writeGhosts ) { + const int& stripe, + const bool writeGhosts) { double allStart = MPI_Wtime(); bool success = true; int myRank; @@ -1067,7 +1100,6 @@ bool writeGrid(dccrg::Dccrg& mpiGrid, const int masterProcessId = 0; MPI_Info MPIinfo; - if (P::systemWriteHints.size() == 0) { MPIinfo = MPI_INFO_NULL; } else { @@ -1080,6 +1112,18 @@ bool writeGrid(dccrg::Dccrg& mpiGrid, MPI_Info_set(MPIinfo, it->first.c_str(), it->second.c_str()); } } + if (stripe == 0 || stripe < -1){ + MPIinfo = MPI_INFO_NULL; + } else { + if ( MPIinfo == MPI_INFO_NULL ) { + MPI_Info_create(&MPIinfo); + } + char stripeChar[6]; + sprintf(stripeChar,"%d",stripe); + /* no. of I/O devices to be used for file striping */ + char factor[] = "striping_factor"; + MPI_Info_set(MPIinfo, factor, stripeChar); + } phiprof::start("open"); vlsvWriter.open( fname.str(), MPI_COMM_WORLD, masterProcessId, MPIinfo ); @@ -1199,16 +1243,16 @@ bool writeGrid(dccrg::Dccrg& mpiGrid, \param fileIndex File index, file will be called "name.index.vlsv" */ bool writeRestart(dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, DataReducer& dataReducer, const string& name, const uint& fileIndex, @@ -1251,7 +1295,7 @@ bool writeRestart(dccrg::Dccrg& mpiGrid, //Open the file with vlsvWriter: Writer vlsvWriter; const int masterProcessId = 0; - MPI_Info MPIinfo; + MPI_Info MPIinfo; if (stripe == 0 || stripe < -1){ MPIinfo = MPI_INFO_NULL; } else { @@ -1329,16 +1373,16 @@ bool writeRestart(dccrg::Dccrg& mpiGrid, // Fsgrid Reducers restartReducer.addOperator(new DRO::DataReductionOperatorFsGrid("fg_E",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]*fsgrids::efield::N_EFIELD); int index=0; @@ -1355,16 +1399,16 @@ bool writeRestart(dccrg::Dccrg& mpiGrid, )); restartReducer.addOperator(new DRO::DataReductionOperatorFsGrid("fg_PERB",[]( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid)->std::vector { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)->std::vector { std::array& gridSize = technicalGrid.getLocalSize(); std::vector retval(gridSize[0]*gridSize[1]*gridSize[2]*fsgrids::bfield::N_BFIELD); int index=0; @@ -1381,7 +1425,7 @@ bool writeRestart(dccrg::Dccrg& mpiGrid, )); //Write necessary variables: - const bool writeAsFloat = false; + const bool writeAsFloat = P::writeRestartAsFloat; for (uint i=0; i #include +#include "definitions.h" #include "spatial_cell.hpp" #include "datareduction/datareducer.h" @@ -41,20 +42,21 @@ \param writeGhosts Write ghost zones */ bool writeGrid(dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, DataReducer* dataReducer, const uint& index, + const int& stripe, const bool writeGhosts = true -); + ); /*! @@ -66,20 +68,21 @@ bool writeGrid(dccrg::Dccrg& mpiGrid, \param fileIndex File index, file will be called "name.index.vlsv" */ bool writeRestart(dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& EGrid, - FsGrid< std::array, 2>& EHallGrid, - FsGrid< std::array, 2>& EGradPeGrid, - FsGrid< std::array, 2>& momentsGrid, - FsGrid< std::array, 2>& dPerBGrid, - FsGrid< std::array, 2>& dMomentsGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< std::array, 2>& volGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & momentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, DataReducer& dataReducer, const std::string& name, const uint& fileIndex, - const int& stripe); + const int& stripe + ); /*! diff --git a/item_storage.h b/item_storage.h index 526aad45b..56ec7c108 100644 --- a/item_storage.h +++ b/item_storage.h @@ -24,6 +24,7 @@ #define ITEM_STORAGE_H #include +#include #include "definitions.h" diff --git a/mesh_data.h b/mesh_data.h index 78d56f1f1..fc2950ad1 100644 --- a/mesh_data.h +++ b/mesh_data.h @@ -29,6 +29,7 @@ #define MESH_DATA_H #include +#include namespace mesh { diff --git a/mini-apps/build_pencils/grid_test.cpp b/mini-apps/build_pencils/grid_test.cpp index 1e07c0088..00782b67a 100644 --- a/mini-apps/build_pencils/grid_test.cpp +++ b/mini-apps/build_pencils/grid_test.cpp @@ -173,8 +173,9 @@ setOfPencils buildPencils( dccrg::Dccrg grid, uint i1 = i + 1; uint id = idsIn[i]; - vector children = grid.get_all_children(id); - bool hasChildren = ( grid.get_parent(children[0]) == id ); + + std::array children = mpiGrid.mapping.get_all_children(id); + bool hasChildren = ( grid.mapping.get_parent(children[0]) == id ); // Check if the current cell contains refined cells if (hasChildren) { @@ -303,12 +304,12 @@ int main(int argc, char* argv[]) { ids.push_back(cell.id); // Add parent cells of refined cells to the list of cell ids. - CellID parent = grid.get_parent(cell.id); + CellID parent = grid.mapping.get_parent(cell.id); if (parent > 0 && !(std::find(ids.begin(), ids.end(), parent) != ids.end())) { ids.push_back(parent); std::cout << "Cell " << parent << " at refinement level " << grid.get_refinement_level(parent) << " has been refined into "; - for (const auto& child: grid.get_all_children(parent)) { + for (const auto& child: grid.mapping.get_all_children(parent)) { std::cout << child << " "; } std::cout << "\n"; diff --git a/mini-apps/build_pencils/grid_test_neighbors.cpp b/mini-apps/build_pencils/grid_test_neighbors.cpp index 8c13fb493..9320ff2ec 100644 --- a/mini-apps/build_pencils/grid_test_neighbors.cpp +++ b/mini-apps/build_pencils/grid_test_neighbors.cpp @@ -132,8 +132,10 @@ setOfPencils buildPencilsWithNeighbors( dccrg::Dccrg &grid, // Maybe you could use physical coordinates here? if( startingRefLvl > path.size() ) { for ( uint i = path.size(); i < startingRefLvl; i++) { - auto parent = grid.get_parent(id); - auto children = grid.get_all_children(parent); + auto parent = grid.mapping.get_parent(id); + + std::array childrenarr = mpiGrid.mapping.get_all_children(parent); + vector children(childrenarr.begin(), childrenarr.end()); auto it = std::find(children.begin(),children.end(),id); auto index = std::distance(children.begin(),it); auto index2 = index; diff --git a/parameters.cpp b/parameters.cpp index ecb60e04c..244e3314b 100644 --- a/parameters.cpp +++ b/parameters.cpp @@ -20,6 +20,7 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ +#include #include #include #include "parameters.h" @@ -67,6 +68,8 @@ Real P::fieldSolverMaxCFL = NAN; Real P::fieldSolverMinCFL = NAN; uint P::fieldSolverSubcycles = 1; +bool P::transShortPencils = true; + uint P::tstep = 0; uint P::tstep_min = 0; uint P::tstep_max = 0; @@ -84,6 +87,8 @@ vector P::systemWriteDistributionWriteStride; vector P::systemWriteDistributionWriteXlineStride; vector P::systemWriteDistributionWriteYlineStride; vector P::systemWriteDistributionWriteZlineStride; +vector P::systemWriteDistributionWriteShellRadius; +vector P::systemWriteDistributionWriteShellStride; vector P::systemWrites; std::vector> P::systemWriteHints; @@ -91,6 +96,7 @@ Real P::saveRestartWalltimeInterval = -1.0; uint P::exitAfterRestarts = numeric_limits::max(); uint64_t P::vlsvBufferSize = 0; int P::restartStripeFactor = -1; +int P::bulkStripeFactor = -1; string P::restartWritePath = string(""); uint P::transmit = 0; @@ -110,10 +116,13 @@ bool P::fieldSolverDiffusiveEterms = true; uint P::ohmHallTerm = 0; uint P::ohmGradPeTerm = 0; Real P::electronTemperature = 0.0; +Real P::electronDensity = 0.0; +Real P::electronPTindex = 1.0; string P::restartFileName = string(""); bool P::isRestart=false; int P::writeAsFloat = false; +int P::writeRestartAsFloat = false; string P::loadBalanceAlgorithm = string(""); string P::loadBalanceTolerance = string(""); uint P::rebalanceInterval = numeric_limits::max(); @@ -134,7 +143,7 @@ Real P::bailout_max_memory = 1073741824.; uint P::amrMaxVelocityRefLevel = 0; Realf P::amrRefineLimit = 1.0; Realf P::amrCoarsenLimit = 0.5; -string P::amrVelRefCriterion = ""; +string P::amrVelRefCriterion = string(""); int P::amrMaxSpatialRefLevel = 0; int P::amrBoxHalfWidthX = 1; int P::amrBoxHalfWidthY = 1; @@ -142,6 +151,10 @@ int P::amrBoxHalfWidthZ = 1; Realf P::amrBoxCenterX = 0.0; Realf P::amrBoxCenterY = 0.0; Realf P::amrBoxCenterZ = 0.0; +vector P::blurPassString; +std::vector P::numPasses; + + bool Parameters::addParameters(){ //the other default parameters we read through the add/get interface @@ -154,6 +167,8 @@ bool Parameters::addParameters(){ Readparameters::addComposing("io.system_write_distribution_xline_stride", "Every this many lines of cells along the x direction write out their velocity space. 0 is none. [Define for all groups.]"); Readparameters::addComposing("io.system_write_distribution_yline_stride", "Every this many lines of cells along the y direction write out their velocity space. 0 is none. [Define for all groups.]"); Readparameters::addComposing("io.system_write_distribution_zline_stride", "Every this many lines of cells along the z direction write out their velocity space. 0 is none. [Define for all groups.]"); + Readparameters::addComposing("io.system_write_distribution_shell_radius", "At cells intersecting spheres with those radii centred at the origin write out their velocity space. 0 is none."); + Readparameters::addComposing("io.system_write_distribution_shell_stride", "Every this many cells for those on selected shells write out their velocity space. 0 is none."); Readparameters::addComposing("io.system_write_mpiio_hint_key", "MPI-IO hint key passed to the non-restart IO. Has to be matched by io.system_write_mpiio_hint_value."); Readparameters::addComposing("io.system_write_mpiio_hint_value", "MPI-IO hint value passed to the non-restart IO. Has to be matched by io.system_write_mpiio_hint_key."); @@ -163,8 +178,11 @@ bool Parameters::addParameters(){ Readparameters::add("io.number_of_restarts","Exit the simulation after certain number of walltime-based restarts.",numeric_limits::max()); Readparameters::add("io.vlsv_buffer_size", "Buffer size passed to VLSV writer (bytes, up to uint64_t), default 0 as this is sensible on sisu", 0); Readparameters::add("io.write_restart_stripe_factor","Stripe factor for restart writing.", -1); + Readparameters::add("io.write_bulk_stripe_factor","Stripe factor for bulk file and initial grid writing.", -1); Readparameters::add("io.write_as_float","If true, write in floats instead of doubles", false); Readparameters::add("io.restart_write_path", "Path to the location where restart files should be written. Defaults to the local directory, also if the specified destination is not writeable.", string("./")); + + Readparameters::add("transShortPencils", "if true, use one-cell pencils", true); Readparameters::add("propagate_field","Propagate magnetic field during the simulation",true); Readparameters::add("propagate_vlasov_acceleration","Propagate distribution functions during the simulation in velocity space. If false, it is propagated with zero length timesteps.",true); @@ -173,6 +191,7 @@ bool Parameters::addParameters(){ Readparameters::add("hallMinimumRho", "Minimum rho value used for the Hall and electron pressure gradient terms in the Lorentz force and in the field solver. Default is very low and has no effect in practice.", 1.0); Readparameters::add("project", "Specify the name of the project to use. Supported to date (20150610): Alfven Diffusion Dispersion Distributions Firehose Flowthrough Fluctuations Harris KHB Larmor Magnetosphere Multipeak Riemann1 Shock Shocktest Template test_fp testHall test_trans VelocityBox verificationLarmor", string("")); + Readparameters::add("restart.write_as_float","If true, write restart fields in floats instead of doubles", false); Readparameters::add("restart.filename","Restart from this vlsv file. No restart if empty file.",string("")); Readparameters::add("gridbuilder.geometry","Simulation geometry XY4D,XZ4D,XY5D,XZ5D,XYZ6D",string("XYZ6D")); @@ -198,7 +217,9 @@ bool Parameters::addParameters(){ Readparameters::add("fieldsolver.diffusiveEterms", "Enable diffusive terms in the computation of E",true); Readparameters::add("fieldsolver.ohmHallTerm", "Enable/choose spatial order of the Hall term in Ohm's law. 0: off, 1: 1st spatial order, 2: 2nd spatial order", 0); Readparameters::add("fieldsolver.ohmGradPeTerm", "Enable/choose spatial order of the electron pressure gradient term in Ohm's law. 0: off, 1: 1st spatial order.", 0); - Readparameters::add("fieldsolver.electronTemperature", "Constant electron temperature to be used for the electron pressure gradient term (K).", 0.0); + Readparameters::add("fieldsolver.electronTemperature", "Upstream electron temperature to be used for the electron pressure gradient term (K).", 0.0); + Readparameters::add("fieldsolver.electronDensity", "Upstream electron density to be used for the electron pressure gradient term (m^-3).", 0.0); + Readparameters::add("fieldsolver.electronPTindex", "Polytropic index for electron pressure gradient term. 0 is isobaric, 1 is isothermal, 1.667 is adiabatic electrons, ", 0.0); Readparameters::add("fieldsolver.maxCFL","The maximum CFL limit for field propagation. Used to set timestep if dynamic_timestep is true.",0.5); Readparameters::add("fieldsolver.minCFL","The minimum CFL limit for field propagation. Used to set timestep if dynamic_timestep is true.",0.4); @@ -216,7 +237,7 @@ bool Parameters::addParameters(){ // Output variable parameters // NOTE Do not remove the : before the list of variable names as this is parsed by tools/check_vlasiator_cfg.sh Readparameters::addComposing("variables.output", std::string()+"List of data reduction operators (DROs) to add to the grid file output. Each variable to be added has to be on a new line output = XXX. Names are case insensitive. "+ - "Available (20190521): "+ + "Available (20210125): "+ "fg_b fg_b_background fg_b_perturbed fg_e "+ "vg_rhom vg_rhoq populations_vg_rho "+ "fg_rhom fg_rhoq "+ @@ -226,7 +247,7 @@ bool Parameters::addParameters(){ "populations_vg_energydensity populations_vg_precipitationdifferentialflux "+ "vg_maxdt_acceleration vg_maxdt_translation populations_vg_maxdt_acceleration populations_vg_maxdt_translation "+ "fg_maxdt_fieldsolver "+ - "vg_rank fg_rank vg_loadbalance_weight "+ + "vg_rank fg_rank fg_amr_level vg_loadbalance_weight "+ "vg_boundarytype fg_boundarytype vg_boundarylayer fg_boundarylayer "+ "populations_vg_blocks vg_f_saved "+ "populations_vg_acceleration_subcycles "+ @@ -260,19 +281,26 @@ bool Parameters::addParameters(){ // NOTE Do not remove the : before the list of variable names as this is parsed by tools/check_vlasiator_cfg.sh Readparameters::addComposing("variables.diagnostic", std::string()+"List of data reduction operators (DROs) to add to the diagnostic runtime output. Each variable to be added has to be on a new line diagnostic = XXX. Names are case insensitive. "+ - "Available (20190320): "+ + "Available (20201111): "+ "populations_vg_blocks "+ - "rhom populations_rho_loss_adjust"+ - "loadbalance_weight"+ - "maxdt_acceleration maxdt_translation populations_maxdt_acceleration populations_maxdt_translation "+ - "maxdt_fieldsolver "+ - "populations_maxdistributionfunction populations_mindistributionfunction"); + "vg_rhom populations_vg_rho_loss_adjust "+ + "vg_loadbalance_weight "+ + "vg_maxdt_acceleration vg_maxdt_translation "+ + "fg_maxdt_fieldsolver "+ + "populations_vg_maxdt_acceleration populations_vg_maxdt_translation "+ + "populations_vg_maxdistributionfunction populations_vg_mindistributionfunction"); Readparameters::addComposing("variables_deprecated.diagnostic", std::string()+"List of deprecated data reduction operators (DROs) to add to the diagnostic runtime output. Names are case insensitive. "+ - "Available (20190320): "+ - "populations_rholossadjust"+ - "LBweight"+ - "populations_MaxVdt MaxVdt populations_MaxRdt MaxRdt MaxFieldsdt"); + "Available (20201111): "+ + "rhom populations_rholossadjust populations_rho_loss_adjust "+ + "populations_blocks lbweight loadbalance_weight "+ + "vg_lbweight vg_loadbalanceweight "+ + "maxvdt maxdt_acceleration "+ + "maxrdt maxdt_translation "+ + "populations_maxvdt populations_maxrdt "+ + "populations_maxdt_acceleration populations_maxdt_translation "+ + "populations_maxdistributionfunction populations_mindistributionfunction "+ + "maxfieldsdt maxdt_fieldsolver fg_maxfieldsdt"); // bailout parameters Readparameters::add("bailout.write_restart", "If 1, write a restart file on bailout. Gets reset when sending a STOP (1) or a KILL (0).", true); @@ -291,6 +319,7 @@ bool Parameters::addParameters(){ Readparameters::add("AMR.box_center_x","x coordinate of the center of the box that is refined (for testing)",0.0); Readparameters::add("AMR.box_center_y","y coordinate of the center of the box that is refined (for testing)",0.0); Readparameters::add("AMR.box_center_z","z coordinate of the center of the box that is refined (for testing)",0.0); + Readparameters::addComposing("AMR.filterpasses", std::string("AMR filter passes for each individual refinement level")); return true; } @@ -306,13 +335,17 @@ bool Parameters::getParameters(){ Readparameters::get("io.system_write_distribution_xline_stride", P::systemWriteDistributionWriteXlineStride); Readparameters::get("io.system_write_distribution_yline_stride", P::systemWriteDistributionWriteYlineStride); Readparameters::get("io.system_write_distribution_zline_stride", P::systemWriteDistributionWriteZlineStride); + Readparameters::get("io.system_write_distribution_shell_radius", P::systemWriteDistributionWriteShellRadius); + Readparameters::get("io.system_write_distribution_shell_stride", P::systemWriteDistributionWriteShellStride); Readparameters::get("io.write_initial_state", P::writeInitialState); Readparameters::get("io.restart_walltime_interval", P::saveRestartWalltimeInterval); Readparameters::get("io.number_of_restarts", P::exitAfterRestarts); Readparameters::get("io.vlsv_buffer_size", P::vlsvBufferSize); Readparameters::get("io.write_restart_stripe_factor", P::restartStripeFactor); + Readparameters::get("io.write_bulk_stripe_factor", P::bulkStripeFactor); Readparameters::get("io.restart_write_path", P::restartWritePath); Readparameters::get("io.write_as_float", P::writeAsFloat); + Readparameters::get("transShortPencils", P::transShortPencils); // Checks for validity of io and restart parameters int myRank; @@ -374,6 +407,13 @@ bool Parameters::getParameters(){ } return false; } + if ( P::systemWriteDistributionWriteShellStride.size() != P::systemWriteDistributionWriteShellRadius.size()) { + if(myRank == MASTER_RANK) { + cerr << "ERROR You should set the same number of io.system_write_distribution_shell_stride " << + "and io.system_write_distribution_shell_radius." << endl; + } + return false; + } if ( P::systemWritePath.size() == 0 ) { for (uint i = 0; i < P::systemWriteName.size(); i++) { P::systemWritePath.push_back(string("./")); @@ -411,6 +451,7 @@ bool Parameters::getParameters(){ Readparameters::get("hallMinimumRho",hallRho); P::hallMinimumRhom = hallRho*physicalconstants::MASS_PROTON; P::hallMinimumRhoq = hallRho*physicalconstants::CHARGE; + Readparameters::get("restart.write_as_float", P::writeRestartAsFloat); Readparameters::get("restart.filename",P::restartFileName); P::isRestart=(P::restartFileName!=string("")); @@ -453,7 +494,68 @@ bool Parameters::getParameters(){ Readparameters::get("AMR.vel_refinement_criterion",P::amrVelRefCriterion); Readparameters::get("AMR.refine_limit",P::amrRefineLimit); Readparameters::get("AMR.coarsen_limit",P::amrCoarsenLimit); + + /*Read Blur Passes per Refinement Level*/ + Readparameters::get("AMR.filterpasses",P::blurPassString); + + // Construct Vector of Passes used in grid.cpp + bool isEmpty = blurPassString.size()==0; + std::vector::iterator maxNumPassesPtr; + int maxNumPassesInt; + + if (!isEmpty){ + + for (auto i : blurPassString){ + P::numPasses.push_back(std::stoi(i)); + } + + + // Reverse Sort and Get the maximum number of filter passes + sort(numPasses.begin(), numPasses.end(), greater()); + + // Sanity Check + if (P::numPasses.size()!=P::amrMaxSpatialRefLevel+1){ + cerr<<"Filter Passes="<%d Passes\n",lev,iter); + lev++; + } + } + }else{ + numPasses={0}; + + if(myRank == MASTER_RANK) { + maxNumPassesPtr=std::max_element(P::numPasses.begin(), P::numPasses.end()); + if(maxNumPassesPtr != numPasses.end()){ + maxNumPassesInt = *maxNumPassesPtr; + } else{ + cerr << "Trying to dereference null pointer \t" << " in " << __FILE__ << ":" << __LINE__ << endl; + return false; + } + + + printf("Filtering is off and max number of Passes is = \t %d\n", *max_element(P::numPasses.begin(), P::numPasses.end())); + } + } + + if (geometryString == "XY4D") P::geometry = geometry::XY4D; else if (geometryString == "XZ4D") P::geometry = geometry::XZ4D; else if (geometryString == "XY5D") P::geometry = geometry::XY5D; @@ -494,6 +596,8 @@ bool Parameters::getParameters(){ Readparameters::get("fieldsolver.ohmHallTerm", P::ohmHallTerm); Readparameters::get("fieldsolver.ohmGradPeTerm", P::ohmGradPeTerm); Readparameters::get("fieldsolver.electronTemperature", P::electronTemperature); + Readparameters::get("fieldsolver.electronDensity", P::electronDensity); + Readparameters::get("fieldsolver.electronPTindex", P::electronPTindex); Readparameters::get("fieldsolver.maxCFL",P::fieldSolverMaxCFL); Readparameters::get("fieldsolver.minCFL",P::fieldSolverMinCFL); // Get Vlasov solver parameters diff --git a/parameters.h b/parameters.h index 867eb37f9..8f519017a 100644 --- a/parameters.h +++ b/parameters.h @@ -58,7 +58,8 @@ struct Parameters { static Real fieldSolverMinCFL; /*!< The minimum CFL limit for propagation of fields. Used to set timestep if useCFLlimit is true.*/ static Real fieldSolverMaxCFL; /*!< The maximum CFL limit for propagation of fields. Used to set timestep if useCFLlimit is true.*/ static uint fieldSolverSubcycles; /*!< The number of field solver subcycles to compute.*/ - + static bool transShortPencils; /*!< Use short or longpencils in AMR translation.*/ + static uint tstep_min; /*!< Timestep when simulation starts, needed for restarts.*/ static uint tstep_max; /*!< Maximum timestep. */ static uint tstep; /*!< The number of the current timestep. 0=initial state. */ @@ -74,6 +75,8 @@ struct Parameters { static std::vector systemWriteDistributionWriteXlineStride; /*!< Every this many lines of cells along the x direction write out their velocity space in each class. */ static std::vector systemWriteDistributionWriteYlineStride; /*!< Every this many lines of cells along the y direction write out their velocity space in each class. */ static std::vector systemWriteDistributionWriteZlineStride; /*!< Every this many lines of cells along the z direction write out their velocity space in each class. */ + static std::vector systemWriteDistributionWriteShellRadius; /*!< At cells intersecting spheres with those radii centred at the origin write out their velocity space in each class. */ + static std::vector systemWriteDistributionWriteShellStride; /*!< Every this many cells for those on selected shells write out their velocity space in each class. */ static std::vector systemWrites; /*!< How many files have been written of each class*/ static std::vector> systemWriteHints; /*!< Collection of MPI-IO hints passed for non-restart IO. Pairs of key-value strings. */ @@ -82,6 +85,7 @@ struct Parameters { static uint exitAfterRestarts; /*!< Exit after this many restarts*/ static uint64_t vlsvBufferSize; /*!< Buffer size in bytes passed to VLSV writer. */ static int restartStripeFactor; /*!< stripe_factor for restart writing*/ + static int bulkStripeFactor; /*!< stripe_factor for bulk and initial grid writing*/ static std::string restartWritePath; /*!< Path to the location where restart files should be written. Defaults to the local directory, also if the specified destination is not writeable. */ static uint transmit; @@ -100,7 +104,10 @@ struct Parameters { static Real resistivity; /*!< Resistivity in Ohm's law eta*J term. */ static uint ohmHallTerm; /*!< Enable/choose spatial order of Hall term in Ohm's law JXB term. 0: off, 1: 1st spatial order, 2: 2nd spatial order. */ static uint ohmGradPeTerm; /*!< Enable/choose spatial order of the electron pressure gradient term in Ohm's law. 0: off, 1: 1st spatial order. */ - static Real electronTemperature; /*!< Constant electron temperature to be used for the electron pressure gradient term (K). */ + static Real electronTemperature; /*!< Upstream electron temperature to be used for the electron pressure gradient term (K). */ + static Real electronDensity; /*!< Upstream electron density to be used for the electron pressure gradient term (m^-3). */ + static Real electronPTindex; /*!> Polytropic index for electron pressure gradient term. 0 is isobaric, 1 is isothermal, 1.667 is adiabatic electrons */ + static bool fieldSolverDiffusiveEterms; /*!< Enable resistive terms in the computation of E*/ static Real maxSlAccelerationRotation; /*!< Maximum rotation in acceleration for semilagrangian solver*/ @@ -121,6 +128,7 @@ struct Parameters { static std::string restartFileName; /*!< If defined, restart from this file*/ static bool isRestart; /*!< true if this is a restart, false otherwise */ static int writeAsFloat; /*!< true if writing into VLSV in floats instead of doubles, false otherwise */ + static int writeRestartAsFloat; /*!< true if writing into restart files in floats instead of doubles, false otherwise */ static bool dynamicTimestep; /*!< If true, timestep is set based on CFL limit */ static std::string projectName; /*!< Project to be used in this run. */ @@ -142,8 +150,10 @@ struct Parameters { static Realf amrBoxCenterX; static Realf amrBoxCenterY; static Realf amrBoxCenterZ; + static std::vector blurPassString; + static std::vector numPasses; - /*! \brief Add the global parameters. + /*! \brief Add the global parameters. * * This function adds all the parameters that are loaded at a global level. * More are being loaded e.g. in the projects and in the system boundary conditions. diff --git a/projects/Alfven/Alfven.cpp b/projects/Alfven/Alfven.cpp index 1a48bec20..e7fcaade9 100644 --- a/projects/Alfven/Alfven.cpp +++ b/projects/Alfven/Alfven.cpp @@ -163,9 +163,9 @@ namespace projects { } void Alfven::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { setBackgroundFieldToZero(BgBGrid); diff --git a/projects/Alfven/Alfven.h b/projects/Alfven/Alfven.h index 69642ca7f..81c6fba53 100644 --- a/projects/Alfven/Alfven.h +++ b/projects/Alfven/Alfven.h @@ -44,9 +44,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: diff --git a/projects/Diffusion/Diffusion.cpp b/projects/Diffusion/Diffusion.cpp index ea92c60ad..b3bd28c36 100644 --- a/projects/Diffusion/Diffusion.cpp +++ b/projects/Diffusion/Diffusion.cpp @@ -122,9 +122,9 @@ namespace projects { void Diffusion::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } void Diffusion::setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(0,0,this->B0); //bg bx, by,bz diff --git a/projects/Diffusion/Diffusion.h b/projects/Diffusion/Diffusion.h index 007c7fbab..ab6a4b089 100644 --- a/projects/Diffusion/Diffusion.h +++ b/projects/Diffusion/Diffusion.h @@ -47,9 +47,9 @@ namespace projects { virtual void getParameters(void); /*! set background field, should set it for all cells */ virtual void setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: diff --git a/projects/Dispersion/Dispersion.cpp b/projects/Dispersion/Dispersion.cpp index 1034f9f3c..28cfe85d2 100644 --- a/projects/Dispersion/Dispersion.cpp +++ b/projects/Dispersion/Dispersion.cpp @@ -205,9 +205,9 @@ namespace projects { } void Dispersion::setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(this->B0 * cos(this->angleXY) * cos(this->angleXZ), diff --git a/projects/Dispersion/Dispersion.h b/projects/Dispersion/Dispersion.h index 3a1d5b902..3d68ce037 100644 --- a/projects/Dispersion/Dispersion.h +++ b/projects/Dispersion/Dispersion.h @@ -52,9 +52,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); virtual void hook( cuint& stage, diff --git a/projects/Distributions/Distributions.cpp b/projects/Distributions/Distributions.cpp index 7c68241f7..b7f03b2d0 100644 --- a/projects/Distributions/Distributions.cpp +++ b/projects/Distributions/Distributions.cpp @@ -157,9 +157,9 @@ namespace projects { } void Distributions::setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(this->Bx, diff --git a/projects/Distributions/Distributions.h b/projects/Distributions/Distributions.h index b594caf9b..951820d60 100644 --- a/projects/Distributions/Distributions.h +++ b/projects/Distributions/Distributions.h @@ -37,9 +37,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: Real getDistribValue( diff --git a/projects/Firehose/Firehose.cpp b/projects/Firehose/Firehose.cpp index 77641034b..94d7859c5 100644 --- a/projects/Firehose/Firehose.cpp +++ b/projects/Firehose/Firehose.cpp @@ -154,9 +154,9 @@ namespace projects { void Firehose::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } void Firehose::setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(this->Bx, diff --git a/projects/Firehose/Firehose.h b/projects/Firehose/Firehose.h index 97a6682cc..fc17179c7 100644 --- a/projects/Firehose/Firehose.h +++ b/projects/Firehose/Firehose.h @@ -50,9 +50,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: Real getDistribValue( diff --git a/projects/Flowthrough/Flowthrough.cpp b/projects/Flowthrough/Flowthrough.cpp index 9deb584d8..379a62f6c 100644 --- a/projects/Flowthrough/Flowthrough.cpp +++ b/projects/Flowthrough/Flowthrough.cpp @@ -41,7 +41,10 @@ using namespace std; enum DensityModel { Maxwellian, - SheetMaxwellian + SheetMaxwellian, + Square, + Triangle, + Sinewave }; static DensityModel densityModel; @@ -58,6 +61,7 @@ namespace projects { typedef Readparameters RP; RP::add("Flowthrough.emptyBox","Is the simulation domain empty initially?",false); RP::add("Flowthrough.densityModel","Plasma density model, 'Maxwellian' or 'SheetMaxwellian'",string("Maxwellian")); + RP::add("Flowthrough.densityWidth","Width of signal around origin",6.e7); RP::add("Flowthrough.Bx", "Magnetic field x component (T)", 0.0); RP::add("Flowthrough.By", "Magnetic field y component (T)", 0.0); RP::add("Flowthrough.Bz", "Magnetic field z component (T)", 0.0); @@ -66,6 +70,7 @@ namespace projects { for(uint i=0; i< getObjectWrapper().particleSpecies.size(); i++) { const std::string& pop = getObjectWrapper().particleSpecies[i].name; RP::add(pop + "_Flowthrough.rho", "Number density (m^-3)", 0.0); + RP::add(pop + "_Flowthrough.rhoBase", "Background number density (m^-3)", 0.0); RP::add(pop + "_Flowthrough.T", "Temperature (K)", 0.0); RP::add(pop + "_Flowthrough.VX0", "Initial bulk velocity in x-direction", 0.0); RP::add(pop + "_Flowthrough.VY0", "Initial bulk velocity in y-direction", 0.0); @@ -104,10 +109,17 @@ namespace projects { } if (densityModelString == "Maxwellian") densityModel = Maxwellian; else if (densityModelString == "SheetMaxwellian") densityModel = SheetMaxwellian; + else if (densityModelString == "Square") densityModel = Square; + else if (densityModelString == "Triangle") densityModel = Triangle; + else if (densityModelString == "Sinewave") densityModel = Sinewave; else { if (myRank == MASTER_RANK) cerr << __FILE__ << ":" << __LINE__ << " ERROR: Unknown option value!" << endl; exit(1); } + if (!RP::get("Flowthrough.densityWidth",this->densityWidth)) { + if (myRank == MASTER_RANK) cerr << __FILE__ << ":" << __LINE__ << " ERROR: This option has not been added!" << endl; + exit(1); + } // Per-population parameters for(uint i=0; i< getObjectWrapper().particleSpecies.size(); i++) { @@ -118,6 +130,10 @@ namespace projects { if(myRank == MASTER_RANK) cerr << __FILE__ << ":" << __LINE__ << " ERROR: This option has not been added for population " << pop << "!" << endl; exit(1); } + if(!RP::get(pop + "_Flowthrough.rhoBase", sP.rhoBase)) { + if(myRank == MASTER_RANK) cerr << __FILE__ << ":" << __LINE__ << " ERROR: This option has not been added for population " << pop << "!" << endl; + exit(1); + } if(!RP::get(pop + "_Flowthrough.T", sP.T)) { if(myRank == MASTER_RANK) cerr << __FILE__ << ":" << __LINE__ << " ERROR: This option has not been added for population " << pop << "!" << endl; exit(1); @@ -163,7 +179,7 @@ namespace projects { break; case SheetMaxwellian: rvalue = sqrt(x*x + y*y + z*z); - if (rvalue <= +3e7) { + if (rvalue <= 0.5*densityWidth) { rvalue = 4*sP.rho * pow(mass / (2.0 * M_PI * physicalconstants::K_B * sP.T), 1.5) * exp(- mass * (( vx-sP.V0[0])*(vx-sP.V0[0]) + (vy-sP.V0[1])*(vy-sP.V0[1]) + (vz-sP.V0[2])*(vz-sP.V0[2])) / (2.0 * physicalconstants::K_B * sP.T)); @@ -171,7 +187,45 @@ namespace projects { rvalue = 0; } break; - } + case Square: + if (abs(x) < 0.5*densityWidth) { + rvalue = 4*sP.rho * pow(mass / (2.0 * M_PI * physicalconstants::K_B * sP.T), 1.5) + * exp(- mass * (( vx-sP.V0[0])*(vx-sP.V0[0]) + (vy-sP.V0[1])*(vy-sP.V0[1]) + + (vz-sP.V0[2])*(vz-sP.V0[2])) / (2.0 * physicalconstants::K_B * sP.T)); + } else { + rvalue = 4*sP.rhoBase * pow(mass / (2.0 * M_PI * physicalconstants::K_B * sP.T), 1.5) + * exp(- mass * (( vx-sP.V0[0])*(vx-sP.V0[0]) + (vy-sP.V0[1])*(vy-sP.V0[1]) + + (vz-sP.V0[2])*(vz-sP.V0[2])) / (2.0 * physicalconstants::K_B * sP.T)); + //rvalue = 0; + } + break; + case Triangle: + if (abs(x) < 0.5*densityWidth) { + rvalue = 4* pow(mass / (2.0 * M_PI * physicalconstants::K_B * sP.T), 1.5) + * exp(- mass * (( vx-sP.V0[0])*(vx-sP.V0[0]) + (vy-sP.V0[1])*(vy-sP.V0[1]) + + (vz-sP.V0[2])*(vz-sP.V0[2])) / (2.0 * physicalconstants::K_B * sP.T)); + rvalue *= ( sP.rhoBase + (sP.rho-sP.rhoBase) * (1.-abs(x) / (0.5*densityWidth))); + } else { + rvalue = 4*sP.rhoBase * pow(mass / (2.0 * M_PI * physicalconstants::K_B * sP.T), 1.5) + * exp(- mass * (( vx-sP.V0[0])*(vx-sP.V0[0]) + (vy-sP.V0[1])*(vy-sP.V0[1]) + + (vz-sP.V0[2])*(vz-sP.V0[2])) / (2.0 * physicalconstants::K_B * sP.T)); + //rvalue = 0; + } + break; + case Sinewave: + if (abs(x) < 0.5*densityWidth) { + rvalue = 4 * pow(mass / (2.0 * M_PI * physicalconstants::K_B * sP.T), 1.5) + * exp(- mass * (( vx-sP.V0[0])*(vx-sP.V0[0]) + (vy-sP.V0[1])*(vy-sP.V0[1]) + + (vz-sP.V0[2])*(vz-sP.V0[2])) / (2.0 * physicalconstants::K_B * sP.T)); + rvalue *= ( sP.rhoBase + (sP.rho-sP.rhoBase) * (0.5 + 0.5*cos(M_PI * x / (0.5*densityWidth)))); + } else { + rvalue = 4*sP.rhoBase * pow(mass / (2.0 * M_PI * physicalconstants::K_B * sP.T), 1.5) + * exp(- mass * (( vx-sP.V0[0])*(vx-sP.V0[0]) + (vy-sP.V0[1])*(vy-sP.V0[1]) + + (vz-sP.V0[2])*(vz-sP.V0[2])) / (2.0 * physicalconstants::K_B * sP.T)); + //rvalue = 0; + } + break; + } return rvalue; } @@ -205,9 +259,9 @@ namespace projects { void Flowthrough::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } void Flowthrough::setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(Bx,By,Bz); //bg bx, by,bz diff --git a/projects/Flowthrough/Flowthrough.h b/projects/Flowthrough/Flowthrough.h index 99ed4d73a..6ef20ad5f 100644 --- a/projects/Flowthrough/Flowthrough.h +++ b/projects/Flowthrough/Flowthrough.h @@ -33,6 +33,7 @@ namespace projects { struct FlowthroughSpeciesParameters { Real rho; + Real rhoBase; Real T; Real V0[3]; uint nSpaceSamples; @@ -48,9 +49,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: @@ -77,6 +78,7 @@ namespace projects { bool emptyBox; /**< If true, then the simulation domain is empty initially * and matter will flow in only through the boundaries.*/ + Real densityWidth; Real Bx; Real By; Real Bz; diff --git a/projects/Fluctuations/Fluctuations.cpp b/projects/Fluctuations/Fluctuations.cpp index 8b00deef0..131ae1704 100644 --- a/projects/Fluctuations/Fluctuations.cpp +++ b/projects/Fluctuations/Fluctuations.cpp @@ -171,9 +171,9 @@ namespace projects { } void Fluctuations::setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(this->BX0, diff --git a/projects/Fluctuations/Fluctuations.h b/projects/Fluctuations/Fluctuations.h index 26c64a3cd..b8d1da48a 100644 --- a/projects/Fluctuations/Fluctuations.h +++ b/projects/Fluctuations/Fluctuations.h @@ -49,9 +49,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); virtual std::vector > getV0( creal x, diff --git a/projects/Harris/Harris.cpp b/projects/Harris/Harris.cpp index e02c1d455..3541b77fa 100644 --- a/projects/Harris/Harris.cpp +++ b/projects/Harris/Harris.cpp @@ -149,9 +149,9 @@ namespace projects { } void Harris::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { setBackgroundFieldToZero(BgBGrid); diff --git a/projects/Harris/Harris.h b/projects/Harris/Harris.h index 3cae54111..2ddd7cb6e 100644 --- a/projects/Harris/Harris.h +++ b/projects/Harris/Harris.h @@ -45,9 +45,9 @@ namespace projects { virtual void getParameters(void); virtual void calcCellParameters(spatial_cell::SpatialCell* cell,creal& t); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); virtual Real calcPhaseSpaceDensity( creal& x, creal& y, creal& z, diff --git a/projects/IPShock/IPShock.cpp b/projects/IPShock/IPShock.cpp index 3fe151245..f5f6fdbb5 100644 --- a/projects/IPShock/IPShock.cpp +++ b/projects/IPShock/IPShock.cpp @@ -31,8 +31,6 @@ Previous development version name was UtuShock #include #include -#include "vectorclass.h" -#include "vector3d.h" #include "../../common.h" #include "../../readparameters.h" @@ -460,9 +458,9 @@ namespace projects { } void IPShock::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { setBackgroundFieldToZero(BgBGrid); diff --git a/projects/IPShock/IPShock.h b/projects/IPShock/IPShock.h index aae3e3b4b..ffed1db31 100644 --- a/projects/IPShock/IPShock.h +++ b/projects/IPShock/IPShock.h @@ -63,9 +63,9 @@ namespace projects { virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); virtual Real calcPhaseSpaceDensity( creal& x, creal& y, creal& z, diff --git a/projects/KHB/KHB.cpp b/projects/KHB/KHB.cpp index 5698d38b4..d4f332686 100644 --- a/projects/KHB/KHB.cpp +++ b/projects/KHB/KHB.cpp @@ -154,9 +154,9 @@ namespace projects { void KHB::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } void KHB::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { setBackgroundFieldToZero(BgBGrid); diff --git a/projects/KHB/KHB.h b/projects/KHB/KHB.h index a11332cc1..0a1215b39 100644 --- a/projects/KHB/KHB.h +++ b/projects/KHB/KHB.h @@ -46,9 +46,9 @@ namespace projects { const uint popID ) const; virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: Real getDistribValue( diff --git a/projects/Larmor/Larmor.cpp b/projects/Larmor/Larmor.cpp index ea6ed8bb6..bcc9c4ce4 100644 --- a/projects/Larmor/Larmor.cpp +++ b/projects/Larmor/Larmor.cpp @@ -141,9 +141,9 @@ namespace projects { void Larmor::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } void Larmor::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(this->BX0, diff --git a/projects/Larmor/Larmor.h b/projects/Larmor/Larmor.h index 69dbc48d5..5bbce55b4 100644 --- a/projects/Larmor/Larmor.h +++ b/projects/Larmor/Larmor.h @@ -39,9 +39,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: Real getDistribValue( diff --git a/projects/Magnetosphere/Magnetosphere.cpp b/projects/Magnetosphere/Magnetosphere.cpp index a2d2bba95..b7090a8ea 100644 --- a/projects/Magnetosphere/Magnetosphere.cpp +++ b/projects/Magnetosphere/Magnetosphere.cpp @@ -76,6 +76,10 @@ namespace projects { RP::add("Magnetosphere.dipoleInflowBX","Inflow magnetic field Bx component to which the vector potential dipole converges. Default is none.", 0.0); RP::add("Magnetosphere.dipoleInflowBY","Inflow magnetic field By component to which the vector potential dipole converges. Default is none.", 0.0); RP::add("Magnetosphere.dipoleInflowBZ","Inflow magnetic field Bz component to which the vector potential dipole converges. Default is none.", 0.0); + //New Parameter for zeroing out derivativeNew Parameter for zeroing out derivativess + RP::add("Magnetosphere.zeroOutDerivativesX","Zero Out Perpendicular components", 1.0); + RP::add("Magnetosphere.zeroOutDerivativesY","Zero Out Perpendicular components", 1.0); + RP::add("Magnetosphere.zeroOutDerivativesZ","Zero Out Perpendicular components", 1.0); // Per-population parameters for(uint i=0; i< getObjectWrapper().particleSpecies.size(); i++) { @@ -231,6 +235,20 @@ namespace projects { exit(1); } + if(!Readparameters::get("Magnetosphere.zeroOutDerivativesX", this->zeroOutComponents[0])) { + if(myRank == MASTER_RANK) cerr << __FILE__ << ":" << __LINE__ << " ERROR: This option has not been added!" << endl; + exit(1); + } + + if(!Readparameters::get("Magnetosphere.zeroOutDerivativesY", this->zeroOutComponents[1])) { + if(myRank == MASTER_RANK) cerr << __FILE__ << ":" << __LINE__ << " ERROR: This option has not been added!" << endl; + exit(1); + } + + if(!Readparameters::get("Magnetosphere.zeroOutDerivativesZ", this->zeroOutComponents[2])) { + if(myRank == MASTER_RANK) cerr << __FILE__ << ":" << __LINE__ << " ERROR: This option has not been added!" << endl; + exit(1); + } // Per-population parameters for(uint i=0; i< getObjectWrapper().particleSpecies.size(); i++) { const std::string& pop = getObjectWrapper().particleSpecies[i].name; @@ -334,9 +352,9 @@ namespace projects { /* set 0-centered dipole */ void Magnetosphere::setProjectBField( - FsGrid< std::array, 2>& perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { Dipole bgFieldDipole; LineDipole bgFieldLineDipole; @@ -378,7 +396,7 @@ namespace projects { if (P::isRestart == false) { bgFieldDipole.initialize(-8e15 *this->dipoleScalingFactor, 0.0, 0.0, 0.0, 0.0 ); setPerturbedField(bgFieldDipole, perBGrid); - bgVectorDipole.initialize(8e15 *this->dipoleScalingFactor, 0.0, 0.0, 0.0, this->dipoleTiltPhi*3.14159/180., this->dipoleTiltTheta*3.14159/180., this->dipoleXFull, this->dipoleXZero, this->dipoleInflowB[0], this->dipoleInflowB[1], this->dipoleInflowB[2]); + bgVectorDipole.initialize(8e15 *this->dipoleScalingFactor, 0.0, 0.0, 0.0, this->dipoleTiltPhi*M_PI/180., this->dipoleTiltTheta*M_PI/180., this->dipoleXFull, this->dipoleXZero, this->dipoleInflowB[0], this->dipoleInflowB[1], this->dipoleInflowB[2]); setPerturbedField(bgVectorDipole, perBGrid, true); } break; @@ -390,8 +408,11 @@ namespace projects { #pragma omp parallel { + bool doZeroOut; //Force field to zero in the perpendicular direction for 2D (1D) simulations. Otherwise we have unphysical components. - if(P::xcells_ini==1) { + doZeroOut = P::xcells_ini ==1 && this->zeroOutComponents[0]==1; + + if(doZeroOut) { #pragma omp for collapse(3) for (int x = 0; x < localSize[0]; ++x) { for (int y = 0; y < localSize[1]; ++y) { @@ -411,28 +432,32 @@ namespace projects { } } } - if(P::ycells_ini==1) { - /*2D simulation in x and z. Set By and derivatives along Y, and derivatives of By to zero*/ -#pragma omp for collapse(3) - for (int x = 0; x < localSize[0]; ++x) { - for (int y = 0; y < localSize[1]; ++y) { - for (int z = 0; z < localSize[2]; ++z) { - std::array* cell = BgBGrid.get(x, y, z); - cell->at(fsgrids::bgbfield::BGBY)=0.0; - cell->at(fsgrids::bgbfield::BGBYVOL)=0.0; - cell->at(fsgrids::bgbfield::dBGBxdy)=0.0; - cell->at(fsgrids::bgbfield::dBGBzdy)=0.0; - cell->at(fsgrids::bgbfield::dBGBydx)=0.0; - cell->at(fsgrids::bgbfield::dBGBydz)=0.0; - cell->at(fsgrids::bgbfield::dBGBXVOLdy)=0.0; - cell->at(fsgrids::bgbfield::dBGBZVOLdy)=0.0; - cell->at(fsgrids::bgbfield::dBGBYVOLdx)=0.0; - cell->at(fsgrids::bgbfield::dBGBYVOLdz)=0.0; - } - } - } - } - if(P::zcells_ini==1) { + + doZeroOut = P::ycells_ini ==1 && this->zeroOutComponents[1]==1; + if(doZeroOut) { + /*2D simulation in x and z. Set By and derivatives along Y, and derivatives of By to zero*/ + #pragma omp for collapse(3) + for (int x = 0; x < localSize[0]; ++x) { + for (int y = 0; y < localSize[1]; ++y) { + for (int z = 0; z < localSize[2]; ++z) { + std::array* cell = BgBGrid.get(x, y, z); + cell->at(fsgrids::bgbfield::BGBY)=0.0; + cell->at(fsgrids::bgbfield::BGBYVOL)=0.0; + cell->at(fsgrids::bgbfield::dBGBxdy)=0.0; + cell->at(fsgrids::bgbfield::dBGBzdy)=0.0; + cell->at(fsgrids::bgbfield::dBGBydx)=0.0; + cell->at(fsgrids::bgbfield::dBGBydz)=0.0; + cell->at(fsgrids::bgbfield::dBGBXVOLdy)=0.0; + cell->at(fsgrids::bgbfield::dBGBZVOLdy)=0.0; + cell->at(fsgrids::bgbfield::dBGBYVOLdx)=0.0; + cell->at(fsgrids::bgbfield::dBGBYVOLdz)=0.0; + } + } + } + } + + doZeroOut = P::zcells_ini ==1 && this->zeroOutComponents[2]==1; + if(doZeroOut) { #pragma omp for collapse(3) for (int x = 0; x < localSize[0]; ++x) { for (int y = 0; y < localSize[1]; ++y) { @@ -465,6 +490,11 @@ namespace projects { for (int i = 0; i < fsgrids::bgbfield::N_BGB; ++i) { BgBGrid.get(x,y,z)->at(i) = 0; } + if ( (this->dipoleType==4) && (P::isRestart == false) ) { + for (int i = 0; i < fsgrids::bfield::N_BFIELD; ++i) { + perBGrid.get(x,y,z)->at(i) = 0; + } + } } } } @@ -601,15 +631,16 @@ namespace projects { if(myRank == MASTER_RANK) std::cout << "Maximum refinement level is " << mpiGrid.mapping.get_maximum_refinement_level() << std::endl; // Leave boundary cells and a bit of safety margin - const int bw = 2* VLASOV_STENCIL_WIDTH; - const int bw2 = 2*(bw + VLASOV_STENCIL_WIDTH); - const int bw3 = 2*(bw2 + VLASOV_STENCIL_WIDTH); - const int bw4 = 2*(bw3 + VLASOV_STENCIL_WIDTH); + const int bw = 2* (globalflags::AMRstencilWidth); + const int bw2 = 2*(bw + globalflags::AMRstencilWidth); + const int bw3 = 2*(bw2 + globalflags::AMRstencilWidth); + const int bw4 = 2*(bw3 + globalflags::AMRstencilWidth); // Calculate regions for refinement if (P::amrMaxSpatialRefLevel > 0) { // L1 refinement. +//#pragma omp parallel for collapse(3) for (uint i = bw; i < P::xcells_ini-bw; ++i) { for (uint j = bw; j < P::ycells_ini-bw; ++j) { for (uint k = bw; k < P::zcells_ini-bw; ++k) { @@ -644,6 +675,7 @@ namespace projects { if (P::amrMaxSpatialRefLevel > 1) { // L2 refinement. +//#pragma omp parallel for collapse(3) for (uint i = bw2; i < 2*P::xcells_ini-bw2; ++i) { for (uint j = bw2; j < 2*P::ycells_ini-bw2; ++j) { for (uint k = bw2; k < 2*P::zcells_ini-bw2; ++k) { @@ -679,6 +711,7 @@ namespace projects { if (P::amrMaxSpatialRefLevel > 2) { // L3 refinement. +//#pragma omp parallel for collapse(3) for (uint i = bw3; i < 4*P::xcells_ini-bw3; ++i) { for (uint j = bw3; j < 4*P::ycells_ini-bw3; ++j) { for (uint k = bw3; k < 4*P::zcells_ini-bw3; ++k) { @@ -730,6 +763,7 @@ namespace projects { if (P::amrMaxSpatialRefLevel > 3) { // L4 refinement. +//#pragma omp parallel for collapse(3) for (uint i = bw4; i < 8*P::xcells_ini-bw4; ++i) { for (uint j = bw4; j < 8*P::ycells_ini-bw4; ++j) { for (uint k = bw4; k < 8*P::zcells_ini-bw4; ++k) { diff --git a/projects/Magnetosphere/Magnetosphere.h b/projects/Magnetosphere/Magnetosphere.h index d6978cc9e..e028f7a4d 100644 --- a/projects/Magnetosphere/Magnetosphere.h +++ b/projects/Magnetosphere/Magnetosphere.h @@ -48,9 +48,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); virtual Real calcPhaseSpaceDensity( creal& x, creal& y, creal& z, @@ -105,6 +105,7 @@ namespace projects { Real dipoleXFull; Real dipoleXZero; Real dipoleInflowB[3]; + Real zeroOutComponents[3]; //0->x,1->y,2->z std::vector speciesParams; }; // class Magnetosphere diff --git a/projects/MultiPeak/MultiPeak.cpp b/projects/MultiPeak/MultiPeak.cpp index ec2a093b8..b774dc484 100644 --- a/projects/MultiPeak/MultiPeak.cpp +++ b/projects/MultiPeak/MultiPeak.cpp @@ -218,9 +218,9 @@ namespace projects { } void MultiPeak::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(this->Bx, diff --git a/projects/MultiPeak/MultiPeak.h b/projects/MultiPeak/MultiPeak.h index 1728141bc..34f5a6bda 100644 --- a/projects/MultiPeak/MultiPeak.h +++ b/projects/MultiPeak/MultiPeak.h @@ -62,9 +62,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: Real getDistribValue( diff --git a/projects/Riemann1/Riemann1.cpp b/projects/Riemann1/Riemann1.cpp index b6372f2a5..0d7154cd6 100644 --- a/projects/Riemann1/Riemann1.cpp +++ b/projects/Riemann1/Riemann1.cpp @@ -120,9 +120,9 @@ namespace projects { void Riemann1::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } void Riemann1::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { setBackgroundFieldToZero(BgBGrid); diff --git a/projects/Riemann1/Riemann1.h b/projects/Riemann1/Riemann1.h index 50f00387c..1da556124 100644 --- a/projects/Riemann1/Riemann1.h +++ b/projects/Riemann1/Riemann1.h @@ -37,9 +37,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: Real getDistribValue( diff --git a/projects/Shock/Shock.cpp b/projects/Shock/Shock.cpp index 2e88b4673..32d38c1d8 100644 --- a/projects/Shock/Shock.cpp +++ b/projects/Shock/Shock.cpp @@ -145,9 +145,9 @@ namespace projects { void Shock::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } void Shock::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { setBackgroundFieldToZero(BgBGrid); diff --git a/projects/Shock/Shock.h b/projects/Shock/Shock.h index b8b1cbf25..b9cd36df4 100644 --- a/projects/Shock/Shock.h +++ b/projects/Shock/Shock.h @@ -36,9 +36,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: diff --git a/projects/Shocktest/Shocktest.cpp b/projects/Shocktest/Shocktest.cpp index 06c8df952..469483dd6 100644 --- a/projects/Shocktest/Shocktest.cpp +++ b/projects/Shocktest/Shocktest.cpp @@ -192,9 +192,9 @@ namespace projects { void Shocktest::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } void Shocktest::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { setBackgroundFieldToZero(BgBGrid); diff --git a/projects/Shocktest/Shocktest.h b/projects/Shocktest/Shocktest.h index a9ceff1ab..6a396a500 100644 --- a/projects/Shocktest/Shocktest.h +++ b/projects/Shocktest/Shocktest.h @@ -63,9 +63,9 @@ namespace projects { const uint popID ) const; virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); virtual void calcCellParameters(spatial_cell::SpatialCell* cell,creal& t); diff --git a/projects/Template/Template.cpp b/projects/Template/Template.cpp index 8f61c0872..c15c2f616 100644 --- a/projects/Template/Template.cpp +++ b/projects/Template/Template.cpp @@ -72,9 +72,9 @@ namespace projects { } void Template::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { Dipole bgField; bgField.initialize(8e15, 0.0, 0.0, 0.0, 0.0); //set dipole moment and location diff --git a/projects/Template/Template.h b/projects/Template/Template.h index ea7612417..bc1082459 100644 --- a/projects/Template/Template.h +++ b/projects/Template/Template.h @@ -36,9 +36,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); virtual Real calcPhaseSpaceDensity( creal& x, creal& y, creal& z, diff --git a/projects/VelocityBox/VelocityBox.cpp b/projects/VelocityBox/VelocityBox.cpp index 891ffbe19..010b8eb73 100644 --- a/projects/VelocityBox/VelocityBox.cpp +++ b/projects/VelocityBox/VelocityBox.cpp @@ -102,9 +102,9 @@ namespace projects { void VelocityBox::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } void VelocityBox::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(this->Bx, diff --git a/projects/VelocityBox/VelocityBox.h b/projects/VelocityBox/VelocityBox.h index 8afbc8134..d04a21a1c 100644 --- a/projects/VelocityBox/VelocityBox.h +++ b/projects/VelocityBox/VelocityBox.h @@ -37,9 +37,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: Real getDistribValue(creal& vx, creal& vy, creal& vz, const uint popID) const; diff --git a/projects/project.cpp b/projects/project.cpp index 150a59fb6..fa0c94fa3 100644 --- a/projects/project.cpp +++ b/projects/project.cpp @@ -180,9 +180,9 @@ namespace projects { /*! Print a warning message to stderr and abort, one should not use the base class functions. */ void Project::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { int rank; MPI_Comm_rank(MPI_COMM_WORLD,&rank); diff --git a/projects/project.h b/projects/project.h index 9765a728a..436449310 100644 --- a/projects/project.h +++ b/projects/project.h @@ -23,6 +23,7 @@ #ifndef PROJECT_H #define PROJECT_H +#include "../definitions.h" #include "../spatial_cell.hpp" #include #include @@ -61,9 +62,9 @@ namespace projects { * \sa setBackgroundField, setBackgroundFieldToZero */ virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); /*! Setup data structures for subsequent setCell calls. diff --git a/projects/projects_common.h b/projects/projects_common.h index 6e83c68c4..e80bcf680 100644 --- a/projects/projects_common.h +++ b/projects/projects_common.h @@ -66,29 +66,4 @@ namespace projects { const uint FACE_NBR_BITMASK = (MISSING_ZNEG | MISSING_YNEG | MISSING_XNEG | MISSING_XPOS | MISSING_YPOS | MISSING_ZPOS); } -// ********************************* -// ***** TEMPLATE DECLARATIONS ***** -// ********************************* - -template bool classifyLevequeGhostCell(const SpatialCell& cell,const CELLID& cellID,const CONT& nbrs); - -template CELLID getNeighbour(const dccrg::Dccrg& mpiGrid,const CELLID& cellID,const int& i,const int& j,const int& k); - - -// ******************************** -// ***** TEMPLATE DEFINITIONS ***** -// ******************************** - -template CELLID getNeighbour(const dccrg::Dccrg& mpiGrid,const CELLID& cellID,const int& i,const int& j,const int& k){ - auto neighbors = mpiGrid.get_neighbors_of_at_offset(cellID, i, j, k); - - //FIXME: support refined grids - if(neighbors.size() > 0) { - return neighbors[0].first; - } else { - return INVALID_CELLID; - } -} - - #endif diff --git a/projects/testAmr/testAmr.cpp b/projects/testAmr/testAmr.cpp index 4dc5442d0..25704efc2 100644 --- a/projects/testAmr/testAmr.cpp +++ b/projects/testAmr/testAmr.cpp @@ -219,9 +219,9 @@ namespace projects { } void testAmr::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(this->Bx, diff --git a/projects/testAmr/testAmr.h b/projects/testAmr/testAmr.h index 19e9c6481..51e295ef7 100644 --- a/projects/testAmr/testAmr.h +++ b/projects/testAmr/testAmr.h @@ -62,9 +62,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: Real getDistribValue( diff --git a/projects/testHall/testHall.cpp b/projects/testHall/testHall.cpp index d8e89a639..bb05da886 100644 --- a/projects/testHall/testHall.cpp +++ b/projects/testHall/testHall.cpp @@ -131,9 +131,9 @@ namespace projects { // cellParams[CellParams::PERBZ ] = this->BZ0 * (x+0.5*Dx)*(y+0.5*Dy)*(z+0.5*Dz)*(x+0.5*Dx)*(y+0.5*Dy)*(z+0.5*Dz)*(x+0.5*Dx)*(y+0.5*Dy)*(z+0.5*Dz); void TestHall::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { setBackgroundFieldToZero(BgBGrid); diff --git a/projects/testHall/testHall.h b/projects/testHall/testHall.h index cf5a1c5b2..15b562319 100644 --- a/projects/testHall/testHall.h +++ b/projects/testHall/testHall.h @@ -37,9 +37,9 @@ namespace projects { virtual void getParameters(void); virtual void calcCellParameters(spatial_cell::SpatialCell* cell,creal& t); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); virtual Real calcPhaseSpaceDensity( creal& x, creal& y, creal& z, diff --git a/projects/test_fp/test_fp.cpp b/projects/test_fp/test_fp.cpp index d9683e4a9..8a401ca47 100644 --- a/projects/test_fp/test_fp.cpp +++ b/projects/test_fp/test_fp.cpp @@ -106,9 +106,9 @@ namespace projects { } void test_fp::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { setBackgroundFieldToZero(BgBGrid); diff --git a/projects/test_fp/test_fp.h b/projects/test_fp/test_fp.h index fe165e619..22adb53f6 100644 --- a/projects/test_fp/test_fp.h +++ b/projects/test_fp/test_fp.h @@ -38,9 +38,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: diff --git a/projects/test_trans/test_trans.cpp b/projects/test_trans/test_trans.cpp index 4ac5a7151..dded499a8 100644 --- a/projects/test_trans/test_trans.cpp +++ b/projects/test_trans/test_trans.cpp @@ -129,9 +129,9 @@ namespace projects { void test_trans::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } void test_trans::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(0.0,0.0,1e-9); diff --git a/projects/test_trans/test_trans.h b/projects/test_trans/test_trans.h index da92bc83d..c56713706 100644 --- a/projects/test_trans/test_trans.h +++ b/projects/test_trans/test_trans.h @@ -38,9 +38,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: diff --git a/projects/verificationLarmor/verificationLarmor.cpp b/projects/verificationLarmor/verificationLarmor.cpp index a919f156e..6d31ab71c 100644 --- a/projects/verificationLarmor/verificationLarmor.cpp +++ b/projects/verificationLarmor/verificationLarmor.cpp @@ -113,9 +113,9 @@ namespace projects { void verificationLarmor::calcCellParameters(spatial_cell::SpatialCell* cell,creal& t) { } void verificationLarmor::setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ) { ConstantField bgField; bgField.initialize(this->BX0, diff --git a/projects/verificationLarmor/verificationLarmor.h b/projects/verificationLarmor/verificationLarmor.h index 2bc34b013..f19dc5547 100644 --- a/projects/verificationLarmor/verificationLarmor.h +++ b/projects/verificationLarmor/verificationLarmor.h @@ -38,9 +38,9 @@ namespace projects { static void addParameters(void); virtual void getParameters(void); virtual void setProjectBField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2>& BgBGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & BgBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid ); protected: Real getDistribValue(creal& vx, creal& vy, creal& vz, const uint popID) const; diff --git a/spatial_cell.cpp b/spatial_cell.cpp index 018035329..94965ca8a 100644 --- a/spatial_cell.cpp +++ b/spatial_cell.cpp @@ -21,7 +21,6 @@ */ #include -#include #include "spatial_cell.hpp" #include "velocity_blocks.h" diff --git a/sysboundary/donotcompute.cpp b/sysboundary/donotcompute.cpp index e22db7ccc..65160191e 100644 --- a/sysboundary/donotcompute.cpp +++ b/sysboundary/donotcompute.cpp @@ -49,13 +49,13 @@ namespace SBC { } bool DoNotCompute::assignSysBoundary(dccrg::Dccrg&, - FsGrid< fsgrids::technical, 2> & technicalGrid) { + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid) { return true; } bool DoNotCompute::applyInitialState( const dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, Project& ) { vector cells = mpiGrid.get_cells(); diff --git a/sysboundary/donotcompute.h b/sysboundary/donotcompute.h index 376298e36..9e55ebecc 100644 --- a/sysboundary/donotcompute.h +++ b/sysboundary/donotcompute.h @@ -49,10 +49,10 @@ namespace SBC { Project &project ); virtual bool assignSysBoundary(dccrg::Dccrg& mpiGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid); + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid); virtual bool applyInitialState( const dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, Project &project ); virtual std::string getName() const; @@ -60,42 +60,38 @@ namespace SBC { // Explicit warning functions to inform the user if a doNotCompute cell gets computed virtual Real fieldSolverBoundaryCondMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, creal& dt, - cuint& RKCase, cuint& component ) { std::cerr << "ERROR: DoNotCompute::fieldSolverBoundaryCondMagneticField called!" << std::endl; return 0.;} virtual void fieldSolverBoundaryCondElectricField( - FsGrid< std::array, 2> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, cint i, cint j, cint k, cuint component ) { std::cerr << "ERROR: DoNotCompute::fieldSolverBoundaryCondElectricField called!" << std::endl;} virtual void fieldSolverBoundaryCondHallElectricField( - FsGrid< std::array, 2> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, cint i, cint j, cint k, cuint component ) { std::cerr << "ERROR: DoNotCompute::fieldSolverBoundaryCondHallElectricField called!" << std::endl;} virtual void fieldSolverBoundaryCondGradPeElectricField( - FsGrid< std::array, 2> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, cint i, cint j, cint k, cuint component ) { std::cerr << "ERROR: DoNotCompute::fieldSolverBoundaryCondGradPeElectricField called!" << std::endl;} virtual void fieldSolverBoundaryCondDerivatives( - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k, @@ -103,7 +99,7 @@ namespace SBC { cuint& component ) { std::cerr << "ERROR: DoNotCompute::fieldSolverBoundaryCondDerivatives called!" << std::endl;} virtual void fieldSolverBoundaryCondBVOLDerivatives( - FsGrid< std::array, 2> & volGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, cint i, cint j, cint k, diff --git a/sysboundary/ionosphere.cpp b/sysboundary/ionosphere.cpp index decad97ca..c7ce426d3 100644 --- a/sysboundary/ionosphere.cpp +++ b/sysboundary/ionosphere.cpp @@ -193,7 +193,7 @@ namespace SBC { } bool Ionosphere::assignSysBoundary(dccrg::Dccrg& mpiGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid) { + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid) { vector cells = mpiGrid.get_cells(); for(uint i=0; isysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE) { @@ -239,7 +239,7 @@ namespace SBC { bool Ionosphere::applyInitialState( const dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, Project &project ) { vector cells = mpiGrid.get_cells(); @@ -255,7 +255,7 @@ namespace SBC { } std::array Ionosphere::fieldSolverGetNormalDirection( - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k @@ -534,54 +534,42 @@ namespace SBC { * -- Retain only the normal components of perturbed face B */ Real Ionosphere::fieldSolverBoundaryCondMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & bGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, creal& dt, - cuint& RKCase, cuint& component ) { - FsGrid< std::array, 2> * bGrid; - - if(RKCase == RK_ORDER1 || RKCase == RK_ORDER2_STEP2) { - bGrid = &perBGrid; - } else { - bGrid = &perBDt2Grid; - } - if (technicalGrid.get(i,j,k)->sysBoundaryLayer == 1) { switch(component) { case 0: if ( ((technicalGrid.get(i-1,j,k)->SOLVE & compute::BX) == compute::BX) && ((technicalGrid.get(i+1,j,k)->SOLVE & compute::BX) == compute::BX) ) { - return 0.5 * (bGrid->get(i-1,j,k)->at(fsgrids::bfield::PERBX) + bGrid->get(i+1,j,k)->at(fsgrids::bfield::PERBX)); + return 0.5 * (bGrid.get(i-1,j,k)->at(fsgrids::bfield::PERBX) + bGrid.get(i+1,j,k)->at(fsgrids::bfield::PERBX)); } else if ((technicalGrid.get(i-1,j,k)->SOLVE & compute::BX) == compute::BX) { - return bGrid->get(i-1,j,k)->at(fsgrids::bfield::PERBX); + return bGrid.get(i-1,j,k)->at(fsgrids::bfield::PERBX); } else if ((technicalGrid.get(i+1,j,k)->SOLVE & compute::BX) == compute::BX) { - return bGrid->get(i+1,j,k)->at(fsgrids::bfield::PERBX); + return bGrid.get(i+1,j,k)->at(fsgrids::bfield::PERBX); } else { Real retval = 0.0; uint nCells = 0; if ((technicalGrid.get(i,j-1,k)->SOLVE & compute::BX) == compute::BX) { - retval += bGrid->get(i,j-1,k)->at(fsgrids::bfield::PERBX); + retval += bGrid.get(i,j-1,k)->at(fsgrids::bfield::PERBX); nCells++; } if ((technicalGrid.get(i,j+1,k)->SOLVE & compute::BX) == compute::BX) { - retval += bGrid->get(i,j+1,k)->at(fsgrids::bfield::PERBX); + retval += bGrid.get(i,j+1,k)->at(fsgrids::bfield::PERBX); nCells++; } if ((technicalGrid.get(i,j,k-1)->SOLVE & compute::BX) == compute::BX) { - retval += bGrid->get(i,j,k-1)->at(fsgrids::bfield::PERBX); + retval += bGrid.get(i,j,k-1)->at(fsgrids::bfield::PERBX); nCells++; } if ((technicalGrid.get(i,j,k+1)->SOLVE & compute::BX) == compute::BX) { - retval += bGrid->get(i,j,k+1)->at(fsgrids::bfield::PERBX); + retval += bGrid.get(i,j,k+1)->at(fsgrids::bfield::PERBX); nCells++; } if (nCells == 0) { @@ -589,7 +577,7 @@ namespace SBC { for (int b=j-1; bSOLVE & compute::BX) == compute::BX) { - retval += bGrid->get(a,b,c)->at(fsgrids::bfield::PERBX); + retval += bGrid.get(a,b,c)->at(fsgrids::bfield::PERBX); nCells++; } } @@ -606,28 +594,28 @@ namespace SBC { if ( (technicalGrid.get(i,j-1,k)->SOLVE & compute::BY) == compute::BY && (technicalGrid.get(i,j+1,k)->SOLVE & compute::BY) == compute::BY ) { - return 0.5 * (bGrid->get(i,j-1,k)->at(fsgrids::bfield::PERBY) + bGrid->get(i,j+1,k)->at(fsgrids::bfield::PERBY)); + return 0.5 * (bGrid.get(i,j-1,k)->at(fsgrids::bfield::PERBY) + bGrid.get(i,j+1,k)->at(fsgrids::bfield::PERBY)); } else if ((technicalGrid.get(i,j-1,k)->SOLVE & compute::BY) == compute::BY) { - return bGrid->get(i,j-1,k)->at(fsgrids::bfield::PERBY); + return bGrid.get(i,j-1,k)->at(fsgrids::bfield::PERBY); } else if ((technicalGrid.get(i,j+1,k)->SOLVE & compute::BY) == compute::BY) { - return bGrid->get(i,j+1,k)->at(fsgrids::bfield::PERBY); + return bGrid.get(i,j+1,k)->at(fsgrids::bfield::PERBY); } else { Real retval = 0.0; uint nCells = 0; if ((technicalGrid.get(i-1,j,k)->SOLVE & compute::BY) == compute::BY) { - retval += bGrid->get(i-1,j,k)->at(fsgrids::bfield::PERBY); + retval += bGrid.get(i-1,j,k)->at(fsgrids::bfield::PERBY); nCells++; } if ((technicalGrid.get(i+1,j,k)->SOLVE & compute::BY) == compute::BY) { - retval += bGrid->get(i+1,j,k)->at(fsgrids::bfield::PERBY); + retval += bGrid.get(i+1,j,k)->at(fsgrids::bfield::PERBY); nCells++; } if ((technicalGrid.get(i,j,k-1)->SOLVE & compute::BY) == compute::BY) { - retval += bGrid->get(i,j,k-1)->at(fsgrids::bfield::PERBY); + retval += bGrid.get(i,j,k-1)->at(fsgrids::bfield::PERBY); nCells++; } if ((technicalGrid.get(i,j,k+1)->SOLVE & compute::BY) == compute::BY) { - retval += bGrid->get(i,j,k+1)->at(fsgrids::bfield::PERBY); + retval += bGrid.get(i,j,k+1)->at(fsgrids::bfield::PERBY); nCells++; } if (nCells == 0) { @@ -635,7 +623,7 @@ namespace SBC { for (int b=j-1; bSOLVE & compute::BY) == compute::BY) { - retval += bGrid->get(a,b,c)->at(fsgrids::bfield::PERBY); + retval += bGrid.get(a,b,c)->at(fsgrids::bfield::PERBY); nCells++; } } @@ -652,28 +640,28 @@ namespace SBC { if ( (technicalGrid.get(i,j,k-1)->SOLVE & compute::BZ) == compute::BZ && (technicalGrid.get(i,j,k+1)->SOLVE & compute::BZ) == compute::BZ ) { - return 0.5 * (bGrid->get(i,j,k-1)->at(fsgrids::bfield::PERBZ) + bGrid->get(i,j,k+1)->at(fsgrids::bfield::PERBZ)); + return 0.5 * (bGrid.get(i,j,k-1)->at(fsgrids::bfield::PERBZ) + bGrid.get(i,j,k+1)->at(fsgrids::bfield::PERBZ)); } else if ((technicalGrid.get(i,j,k-1)->SOLVE & compute::BZ) == compute::BZ) { - return bGrid->get(i,j,k-1)->at(fsgrids::bfield::PERBZ); + return bGrid.get(i,j,k-1)->at(fsgrids::bfield::PERBZ); } else if ((technicalGrid.get(i,j,k+1)->SOLVE & compute::BZ) == compute::BZ) { - return bGrid->get(i,j,k+1)->at(fsgrids::bfield::PERBZ); + return bGrid.get(i,j,k+1)->at(fsgrids::bfield::PERBZ); } else { Real retval = 0.0; uint nCells = 0; if ((technicalGrid.get(i-1,j,k)->SOLVE & compute::BZ) == compute::BZ) { - retval += bGrid->get(i-1,j,k)->at(fsgrids::bfield::PERBZ); + retval += bGrid.get(i-1,j,k)->at(fsgrids::bfield::PERBZ); nCells++; } if ((technicalGrid.get(i+1,j,k)->SOLVE & compute::BZ) == compute::BZ) { - retval += bGrid->get(i+1,j,k)->at(fsgrids::bfield::PERBZ); + retval += bGrid.get(i+1,j,k)->at(fsgrids::bfield::PERBZ); nCells++; } if ((technicalGrid.get(i,j-1,k)->SOLVE & compute::BZ) == compute::BZ) { - retval += bGrid->get(i,j-1,k)->at(fsgrids::bfield::PERBZ); + retval += bGrid.get(i,j-1,k)->at(fsgrids::bfield::PERBZ); nCells++; } if ((technicalGrid.get(i,j+1,k)->SOLVE & compute::BZ) == compute::BZ) { - retval += bGrid->get(i,j+1,k)->at(fsgrids::bfield::PERBZ); + retval += bGrid.get(i,j+1,k)->at(fsgrids::bfield::PERBZ); nCells++; } if (nCells == 0) { @@ -681,7 +669,7 @@ namespace SBC { for (int b=j-1; bSOLVE & compute::BZ) == compute::BZ) { - retval += bGrid->get(a,b,c)->at(fsgrids::bfield::PERBZ); + retval += bGrid.get(a,b,c)->at(fsgrids::bfield::PERBZ); nCells++; } } @@ -705,7 +693,7 @@ namespace SBC { for (int b=j-1; bsysBoundaryLayer == 1) { - retval += bGrid->get(a,b,c)->at(fsgrids::bfield::PERBX + component); + retval += bGrid.get(a,b,c)->at(fsgrids::bfield::PERBX + component); nCells++; } } @@ -720,7 +708,7 @@ namespace SBC { } void Ionosphere::fieldSolverBoundaryCondElectricField( - FsGrid< std::array, 2> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, cint i, cint j, cint k, @@ -730,7 +718,7 @@ namespace SBC { } void Ionosphere::fieldSolverBoundaryCondHallElectricField( - FsGrid< std::array, 2> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, cint i, cint j, cint k, @@ -762,7 +750,7 @@ namespace SBC { } void Ionosphere::fieldSolverBoundaryCondGradPeElectricField( - FsGrid< std::array, 2> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, cint i, cint j, cint k, @@ -772,8 +760,8 @@ namespace SBC { } void Ionosphere::fieldSolverBoundaryCondDerivatives( - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k, @@ -785,7 +773,7 @@ namespace SBC { } void Ionosphere::fieldSolverBoundaryCondBVOLDerivatives( - FsGrid< std::array, 2> & volGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, cint i, cint j, cint k, diff --git a/sysboundary/ionosphere.h b/sysboundary/ionosphere.h index 4f7435b16..fa82563a1 100644 --- a/sysboundary/ionosphere.h +++ b/sysboundary/ionosphere.h @@ -65,49 +65,45 @@ namespace SBC { Project &project ); virtual bool assignSysBoundary(dccrg::Dccrg& mpiGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid); + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid); virtual bool applyInitialState( const dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, Project &project ); virtual Real fieldSolverBoundaryCondMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & bGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, creal& dt, - cuint& RKCase, cuint& component ); virtual void fieldSolverBoundaryCondElectricField( - FsGrid< std::array, 2> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, cint i, cint j, cint k, cuint component ); virtual void fieldSolverBoundaryCondHallElectricField( - FsGrid< std::array, 2> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, cint i, cint j, cint k, cuint component ); virtual void fieldSolverBoundaryCondGradPeElectricField( - FsGrid< std::array, 2> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, cint i, cint j, cint k, cuint component ); virtual void fieldSolverBoundaryCondDerivatives( - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k, @@ -115,7 +111,7 @@ namespace SBC { cuint& component ); virtual void fieldSolverBoundaryCondBVOLDerivatives( - FsGrid< std::array, 2> & volGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, cint i, cint j, cint k, @@ -142,7 +138,7 @@ namespace SBC { ); std::array fieldSolverGetNormalDirection( - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k diff --git a/sysboundary/outflow.cpp b/sysboundary/outflow.cpp index 1df942aae..98d7011a0 100644 --- a/sysboundary/outflow.cpp +++ b/sysboundary/outflow.cpp @@ -215,7 +215,7 @@ namespace SBC { } bool Outflow::assignSysBoundary(dccrg::Dccrg& mpiGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid) { + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid) { bool doAssign; std::array isThisCellOnAFace; @@ -282,7 +282,7 @@ namespace SBC { bool Outflow::applyInitialState( const dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, Project &project ) { const vector& cells = getLocalCells(); @@ -333,30 +333,32 @@ namespace SBC { } Real Outflow::fieldSolverBoundaryCondMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & bGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, creal& dt, - cuint& RKCase, cuint& component ) { - Real fieldValue; - - if(RKCase == RK_ORDER1 || RKCase == RK_ORDER2_STEP2) { - fieldValue = fieldBoundaryCopyFromExistingFaceNbrMagneticField(perBGrid, technicalGrid, i, j, k, component); - } else { - fieldValue = fieldBoundaryCopyFromExistingFaceNbrMagneticField(perBDt2Grid, technicalGrid, i, j, k, component); + switch(component) { + case 0: + return fieldBoundaryCopyFromSolvingNbrMagneticField(bGrid, technicalGrid, i, j, k, component, compute::BX); + break; + case 1: + return fieldBoundaryCopyFromSolvingNbrMagneticField(bGrid, technicalGrid, i, j, k, component, compute::BY); + break; + case 2: + return fieldBoundaryCopyFromSolvingNbrMagneticField(bGrid, technicalGrid, i, j, k, component, compute::BZ); + break; + default: + return 0.0; + break; } - return fieldValue; } void Outflow::fieldSolverBoundaryCondElectricField( - FsGrid< std::array, 2> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, cint i, cint j, cint k, @@ -366,7 +368,7 @@ namespace SBC { } void Outflow::fieldSolverBoundaryCondHallElectricField( - FsGrid< std::array, 2> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, cint i, cint j, cint k, @@ -398,7 +400,7 @@ namespace SBC { } void Outflow::fieldSolverBoundaryCondGradPeElectricField( - FsGrid< std::array, 2> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, cint i, cint j, cint k, @@ -408,8 +410,8 @@ namespace SBC { } void Outflow::fieldSolverBoundaryCondDerivatives( - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k, @@ -420,7 +422,7 @@ namespace SBC { } void Outflow::fieldSolverBoundaryCondBVOLDerivatives( - FsGrid< std::array, 2> & volGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, cint i, cint j, cint k, diff --git a/sysboundary/outflow.h b/sysboundary/outflow.h index eb8985010..7979cd895 100644 --- a/sysboundary/outflow.h +++ b/sysboundary/outflow.h @@ -64,49 +64,45 @@ namespace SBC { Project &project ); virtual bool assignSysBoundary(dccrg::Dccrg& mpiGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid); + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid); virtual bool applyInitialState( const dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, Project &project ); virtual Real fieldSolverBoundaryCondMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & bGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, creal& dt, - cuint& RKCase, cuint& component ); virtual void fieldSolverBoundaryCondElectricField( - FsGrid< std::array, 2> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, cint i, cint j, cint k, cuint component ); virtual void fieldSolverBoundaryCondHallElectricField( - FsGrid< std::array, 2> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, cint i, cint j, cint k, cuint component ); virtual void fieldSolverBoundaryCondGradPeElectricField( - FsGrid< std::array, 2> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, cint i, cint j, cint k, cuint component ); virtual void fieldSolverBoundaryCondDerivatives( - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k, @@ -114,7 +110,7 @@ namespace SBC { cuint& component ); virtual void fieldSolverBoundaryCondBVOLDerivatives( - FsGrid< std::array, 2> & volGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, cint i, cint j, cint k, diff --git a/sysboundary/setbyuser.cpp b/sysboundary/setbyuser.cpp index 8b2a392a5..f8e558532 100644 --- a/sysboundary/setbyuser.cpp +++ b/sysboundary/setbyuser.cpp @@ -79,7 +79,7 @@ namespace SBC { } bool SetByUser::assignSysBoundary(dccrg::Dccrg& mpiGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid) { + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid) { bool doAssign; std::array isThisCellOnAFace; @@ -144,7 +144,7 @@ namespace SBC { bool SetByUser::applyInitialState( const dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, Project &project ) { bool success = true; @@ -157,16 +157,12 @@ namespace SBC { } Real SetByUser::fieldSolverBoundaryCondMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & bGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, creal& dt, - cuint& RKCase, cuint& component ) { Real result = 0.0; @@ -191,7 +187,7 @@ namespace SBC { } void SetByUser::fieldSolverBoundaryCondElectricField( - FsGrid< std::array, 2> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, cint i, cint j, cint k, @@ -201,7 +197,7 @@ namespace SBC { } void SetByUser::fieldSolverBoundaryCondHallElectricField( - FsGrid< std::array, 2> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, cint i, cint j, cint k, @@ -233,7 +229,7 @@ namespace SBC { } void SetByUser::fieldSolverBoundaryCondGradPeElectricField( - FsGrid< std::array, 2> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, cint i, cint j, cint k, @@ -243,8 +239,8 @@ namespace SBC { } void SetByUser::fieldSolverBoundaryCondDerivatives( - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k, @@ -255,7 +251,7 @@ namespace SBC { } void SetByUser::fieldSolverBoundaryCondBVOLDerivatives( - FsGrid< std::array, 2> & volGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, cint i, cint j, cint k, @@ -274,7 +270,7 @@ namespace SBC { } bool SetByUser::setBFromTemplate(const dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid) { + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid) { std::array isThisCellOnAFace; const std::array gridDims(perBGrid.getLocalSize()); diff --git a/sysboundary/setbyuser.h b/sysboundary/setbyuser.h index b10962cf5..0b67f22ba 100644 --- a/sysboundary/setbyuser.h +++ b/sysboundary/setbyuser.h @@ -69,49 +69,45 @@ namespace SBC { Project &project ); virtual bool assignSysBoundary(dccrg::Dccrg& mpiGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid); + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid); virtual bool applyInitialState( const dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, Project &project ); virtual Real fieldSolverBoundaryCondMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & bGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, creal& dt, - cuint& RKCase, cuint& component ); virtual void fieldSolverBoundaryCondElectricField( - FsGrid< std::array, 2> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, cint i, cint j, cint k, cuint component ); virtual void fieldSolverBoundaryCondHallElectricField( - FsGrid< std::array, 2> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, cint i, cint j, cint k, cuint component ); virtual void fieldSolverBoundaryCondGradPeElectricField( - FsGrid< std::array, 2> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, cint i, cint j, cint k, cuint component ); virtual void fieldSolverBoundaryCondDerivatives( - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k, @@ -119,7 +115,7 @@ namespace SBC { cuint& component ); virtual void fieldSolverBoundaryCondBVOLDerivatives( - FsGrid< std::array, 2> & volGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, cint i, cint j, cint k, @@ -146,7 +142,7 @@ namespace SBC { virtual void generateTemplateCell(spatial_cell::SpatialCell& templateCell, Real B[3], int inputDataIndex, creal& t) = 0; bool setCellsFromTemplate(const dccrg::Dccrg& mpiGrid,const uint popID); bool setBFromTemplate(const dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid); + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid); /*! Array of bool telling which faces are going to be processed by the system boundary condition.*/ bool facesToProcess[6]; diff --git a/sysboundary/sysboundary.cpp b/sysboundary/sysboundary.cpp index c55050246..19e870d78 100644 --- a/sysboundary/sysboundary.cpp +++ b/sysboundary/sysboundary.cpp @@ -276,8 +276,10 @@ bool SysBoundary::initSysBoundaries( return success; } -bool SysBoundary::checkRefinement(dccrg::Dccrg& mpiGrid) { - +bool SysBoundary::checkRefinement(dccrg::Dccrg & mpiGrid) { + // Verifies that all cells within FULL_NEIGHBORHOOD_ID of L1 boundary cells are on the same refinement + // level (one group for inner boundary, another for outer boundary) + // Set is used to avoid storing duplicates - each cell only needs to be checked once std::set innerBoundaryCells; std::set outerBoundaryCells; @@ -293,8 +295,8 @@ bool SysBoundary::checkRefinement(dccrg::DccrgsysBoundaryLayer == 1) { - // Add non-boundary neighbors of layer 1 cells - auto* nbrPairVector = mpiGrid.get_neighbors_of(cellId,FULL_NEIGHBORHOOD_ID); + // Add all stencil neighbors of layer 1 cells + auto* nbrPairVector = mpiGrid.get_neighbors_of(cellId,SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID); for (auto nbrPair : *nbrPairVector) { if(nbrPair.first != INVALID_CELLID) { innerBoundaryCells.insert(nbrPair.first); @@ -305,8 +307,8 @@ bool SysBoundary::checkRefinement(dccrg::DccrgsysBoundaryFlag != sysboundarytype::DO_NOT_COMPUTE) { outerBoundaryCells.insert(cellId); outerBoundaryRefLvl = mpiGrid.get_refinement_level(cellId); - // Add non-boundary neighbors of outer boundary cells - auto* nbrPairVector = mpiGrid.get_neighbors_of(cellId,FULL_NEIGHBORHOOD_ID); + // Add all stencil neighbors of outer boundary cells + auto* nbrPairVector = mpiGrid.get_neighbors_of(cellId,SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID); for (auto nbrPair : *nbrPairVector) { if(nbrPair.first != INVALID_CELLID) { outerBoundaryCells.insert(nbrPair.first); @@ -318,13 +320,20 @@ bool SysBoundary::checkRefinement(dccrg::Dccrgparameters[CellParams::XCRD] << ", " << mpiGrid[cellId]->parameters[CellParams::YCRD] << ", " << mpiGrid[cellId]->parameters[CellParams::ZCRD] << + "). Cell level = " << mpiGrid.get_refinement_level(cellId) << + ", boundary level = " << innerBoundaryRefLvl << endl; return false; } } for (auto cellId : outerBoundaryCells) { if (cellId != INVALID_CELLID && mpiGrid.get_refinement_level(cellId) != outerBoundaryRefLvl) { - // cout << "Failed refinement check " << cellId << " " << mpiGrid.get_refinement_level(cellId) << " "<< outerBoundaryRefLvl << endl; + cout << "Failed refinement check (outerBoundary), cellId = " << cellId << + " at (" << mpiGrid[cellId]->parameters[CellParams::XCRD] << ", " << mpiGrid[cellId]->parameters[CellParams::YCRD] << ", " << mpiGrid[cellId]->parameters[CellParams::ZCRD] << + "). Cell level = " << mpiGrid.get_refinement_level(cellId) << + ", boundary level = " << outerBoundaryRefLvl << endl; return false; } } @@ -334,7 +343,7 @@ bool SysBoundary::checkRefinement(dccrg::Dccrg& technicalGrid) { + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid) { bool belongs = false; @@ -374,7 +383,7 @@ bool belongsToLayer(const int layer, const int x, const int y, const int z, * \param mpiGrid Grid */ bool SysBoundary::classifyCells(dccrg::Dccrg& mpiGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid) { + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid) { bool success = true; vector cells = mpiGrid.get_cells(); auto localSize = technicalGrid.getLocalSize().data(); @@ -440,6 +449,9 @@ bool SysBoundary::classifyCells(dccrg::DccrgsysBoundaryLayer==0){ const auto* nbrs = mpiGrid.get_neighbors_of(cells[i],SYSBOUNDARIES_NEIGHBORHOOD_ID); + // Note: this distance calculation will be non-plateau monotonic only assuming that + // SysBoundary::checkRefinement has been applied correctly and there are no refinement + // level changes within SYSBOUNDARIES_NEIGHBORHOOD_ID. for(uint j=0; j<(*nbrs).size(); j++) { if((*nbrs)[j].first!=0 && (*nbrs)[j].first!=cells[i] ) { if(mpiGrid[(*nbrs)[j].first]->sysBoundaryLayer==layer) { @@ -536,9 +548,9 @@ bool SysBoundary::classifyCells(dccrg::Dccrg globalIndices = technicalGrid.getGlobalIndices(x,y,z); - if ( ((globalIndices[0] == 0 || globalIndices[0] == fsGridDimensions[0]-1) && fsGridDimensions[0] > 1) - || ((globalIndices[1] == 0 || globalIndices[1] == fsGridDimensions[1]-1) && fsGridDimensions[1] > 1) - || ((globalIndices[2] == 0 || globalIndices[2] == fsGridDimensions[2]-1) && fsGridDimensions[2] > 1) + if ( ((globalIndices[0] == 0 || globalIndices[0] == fsGridDimensions[0]-1) && !this->isBoundaryPeriodic(0)) + || ((globalIndices[1] == 0 || globalIndices[1] == fsGridDimensions[1]-1) && !this->isBoundaryPeriodic(1)) + || ((globalIndices[2] == 0 || globalIndices[2] == fsGridDimensions[2]-1) && !this->isBoundaryPeriodic(2)) ) { continue; } @@ -593,7 +605,7 @@ bool SysBoundary::classifyCells(dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, Project& project ) { bool success = true; @@ -640,6 +652,7 @@ void SysBoundary::applySysBoundaryVlasovConditions( /*Transfer along boundaries*/ // First the small stuff without overlapping in an extended neighbourhood: + #warning TODO This now communicates in the wider neighbourhood for both layers, could be reduced to smaller neighbourhood for layer 1, larger neighbourhood for layer 2. SpatialCell::set_mpi_transfer_type( Transfer::CELL_PARAMETERS| Transfer::POP_METADATA| @@ -649,12 +662,14 @@ void SysBoundary::applySysBoundaryVlasovConditions( // Loop over existing particle species for (uint popID=0; popID localCells; - getBoundaryCellList(mpiGrid,mpiGrid.get_local_cells_not_on_process_boundary(SYSBOUNDARIES_NEIGHBORHOOD_ID),localCells); + getBoundaryCellList(mpiGrid,mpiGrid.get_local_cells_not_on_process_boundary(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID),localCells); #pragma omp parallel for for (uint i=0; i boundaryCells; - getBoundaryCellList(mpiGrid,mpiGrid.get_local_cells_on_process_boundary(SYSBOUNDARIES_NEIGHBORHOOD_ID),boundaryCells); + getBoundaryCellList(mpiGrid,mpiGrid.get_local_cells_on_process_boundary(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID),boundaryCells); #pragma omp parallel for for (uint i=0; isysBoundaryFlag; @@ -698,12 +713,8 @@ void SysBoundary::applySysBoundaryVlasovConditions( } phiprof::stop(timer); - timer=phiprof::initializeTimer("Wait for sends","MPI","Wait"); - phiprof::start(timer); - mpiGrid.wait_remote_neighbor_copy_update_sends(); - phiprof::stop(timer); - // WARNING Blocks are changed but lists not updated now, if you need to use/communicate them before the next update is done, add an update here. + // reset lists in smaller default neighborhood updateRemoteVelocityBlockLists(mpiGrid, popID); } // for-loop over populations diff --git a/sysboundary/sysboundary.h b/sysboundary/sysboundary.h index 97bf1d8b4..d6b2fd3d9 100644 --- a/sysboundary/sysboundary.h +++ b/sysboundary/sysboundary.h @@ -29,6 +29,7 @@ #include #include #include +#include "../definitions.h" #include "../parameters.h" #include "../readparameters.h" #include "../spatial_cell.hpp" @@ -69,10 +70,10 @@ class SysBoundary { ); bool checkRefinement(dccrg::Dccrg& mpiGrid); bool classifyCells(dccrg::Dccrg& mpiGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid); + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid); bool applyInitialState( dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, Project& project ); void applySysBoundaryVlasovConditions(dccrg::Dccrg& mpiGrid, creal& t, const bool calculate_V_moments); diff --git a/sysboundary/sysboundarycondition.cpp b/sysboundary/sysboundarycondition.cpp index 2b3723455..a326f8da8 100644 --- a/sysboundary/sysboundarycondition.cpp +++ b/sysboundary/sysboundarycondition.cpp @@ -117,8 +117,8 @@ namespace SBC { * \param component 0: x-derivatives, 1: y-derivatives, 2: z-derivatives, 3: xy-derivatives, 4: xz-derivatives, 5: yz-derivatives. */ void SysBoundaryCondition::setCellDerivativesToZero( - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k, @@ -189,7 +189,7 @@ namespace SBC { * \param component 0: x-derivatives, 1: y-derivatives, 2: z-derivatives. */ void SysBoundaryCondition::setCellBVOLDerivativesToZero( - FsGrid< std::array, 2> & volGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, cint i, cint j, cint k, @@ -613,46 +613,58 @@ namespace SBC { std::array & flowtoCells = allFlowtoCells[cellId]; flowtoCells.fill(NULL); uint dist = numeric_limits::max(); - - // First iteration of search to determine closest distance - for(int i=-2; i<3; i++) - for(int j=-2; j<3; j++) - for(int k=-2; k<3; k++) { - const CellID cell = getNeighbour(mpiGrid,cellId,i,j,k); - if(cell != INVALID_CELLID) { - if(mpiGrid[cell]->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) { - cuint d2 = i*i+j*j+k*k; - if(d2 < dist) { - dist = d2; - } - // Flowto neighbours have distances of 1, 2 or 3 at a distance of 1 layer, 4, 5 or 6 at a distance of 2 layers. - // Furthermore one does not want to have the cell itself in this list. - if(d2 < 4 && i != 0 && j != 0 && k != 0) { - flowtoCells.at(i + 3*j + 9*k + 13) = mpiGrid[cell]; - } - if(mpiGrid[cellId]->sysBoundaryLayer == 1 && abs(i) < 2 && abs(j) < 2 && abs(k) < 2) { - closeCells.push_back(cell); - } - if(mpiGrid[cellId]->sysBoundaryLayer == 2) { - closeCells.push_back(cell); - } - } - } - } - // Second iteration to record the cellIds of all cells at closest distance - for(int i=-2; i<3; i++) - for(int j=-2; j<3; j++) - for(int k=-2; k<3; k++) { - const CellID cell = getNeighbour(mpiGrid,cellId,i,j,k); - if(cell != INVALID_CELLID) { - if(mpiGrid[cell]->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) { - cuint d2 = i*i+j*j+k*k; - if(d2 == dist) { - closestCells.push_back(cell); - } - } - } - } + + uint d2 = numeric_limits::max(); + int indexstep = pow(2,P::amrMaxSpatialRefLevel - mpiGrid[cellId]->SpatialCell::parameters[CellParams::REFINEMENT_LEVEL]); + // Note this must be int, not uint, for latter calculations + + // Find flowto cells (note, L2 cells do not have flowto cells) + auto* nearNbrs = mpiGrid.get_neighbors_of(cellId, NEAREST_NEIGHBORHOOD_ID); + for (auto nbrPair : *nearNbrs) { + if(nbrPair.first != INVALID_CELLID) { + if(mpiGrid[nbrPair.first]->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) { + flowtoCells.at((int)(nbrPair.second[0]/indexstep) + 3*(int)(nbrPair.second[1]/indexstep) + + 9*(int)(nbrPair.second[2]/indexstep) + 13) = mpiGrid[nbrPair.first]; + //flowtoCells.at(i + 3*j + 9*k + 13) = mpiGrid[cell]; + } + } + } + // Find all close cells + auto* Nbrs = mpiGrid.get_neighbors_of(cellId, SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID); + for (auto nbrPair : *Nbrs) { + if(nbrPair.first != INVALID_CELLID) { + if(mpiGrid[nbrPair.first]->sysBoundaryFlag == sysboundarytype::NOT_SYSBOUNDARY) { + // Find distance and update closestCells + d2 = nbrPair.second[0]*nbrPair.second[0] + nbrPair.second[1]*nbrPair.second[1] + nbrPair.second[2]*nbrPair.second[2]; + // Only neighboring cells for L1 + if( (mpiGrid[cellId]->sysBoundaryLayer == 1) && + (abs(nbrPair.second[0]) <= indexstep) && + (abs(nbrPair.second[1]) <= indexstep) && + (abs(nbrPair.second[2]) <= indexstep) ) { + closeCells.push_back(nbrPair.first); + if(d2 == dist) { + closestCells.push_back(nbrPair.first); + } else if (d2 < dist) { + closestCells.clear(); + closestCells.push_back(nbrPair.first); + dist = d2; + } + } + // search further for L2 + if(mpiGrid[cellId]->sysBoundaryLayer == 2) { + closeCells.push_back(nbrPair.first); + if(d2 == dist) { + closestCells.push_back(nbrPair.first); + } else if (d2 < dist) { + closestCells.clear(); + closestCells.push_back(nbrPair.first); + dist = d2; + } + } + } + } + } + if(closestCells.size() == 0) closestCells.push_back(INVALID_CELLID); if(closeCells.size() == 0) closeCells.push_back(INVALID_CELLID); } @@ -665,7 +677,7 @@ namespace SBC { * \sa getAllClosestNonsysboundaryCells */ std::array SysBoundaryCondition::getTheClosestNonsysboundaryCell( - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k @@ -680,7 +692,7 @@ namespace SBC { * \sa getTheClosestNonsysboundaryCell */ std::vector< std::array > SysBoundaryCondition::getAllClosestNonsysboundaryCells( - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k @@ -785,48 +797,55 @@ namespace SBC { return flowtoCellsBlock; } - Real SysBoundaryCondition::fieldBoundaryCopyFromExistingFaceNbrMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + Real SysBoundaryCondition::fieldBoundaryCopyFromSolvingNbrMagneticField( + FsGrid< std::array, FS_STENCIL_WIDTH> & bGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, - cuint component + cuint component, + cuint mask ) { - const std::array closestCell = getTheClosestNonsysboundaryCell(technicalGrid, i, j, k); - - const std::array gid = technicalGrid.getGlobalIndices(i, j, k); - const std::array ngid = technicalGrid.getGlobalIndices(closestCell[0], closestCell[1], closestCell[2]); - if (closestCell[0] == std::numeric_limits::min()) { - //cerr << "(" << gid[0] << "," << gid[1] << "," << gid[2] << ")" << __FILE__ << ":" << __LINE__ << ": No closest cell found!" << endl; - //abort(); + int distance = std::numeric_limits::max(); + std::vector< std::array > closestCells; - // When mpiGrid is refined, the fsgrid boundary layer has a width greater than 2. In this case, - // the boundary cells that do not find a non-boundary neighbor just keep their original value, - // we don't care what happens in them since they have no effect on the Vlasov solver. - return perBGrid.get(i,j,k)->at(fsgrids::bfield::PERBX+component); + for (int kk=-2; kk<3; kk++) { + for (int jj=-2; jj<3; jj++) { + for (int ii=-2; ii<3 ; ii++) { + if( technicalGrid.get(i+ii,j+jj,k+kk) // skip invalid cells returning NULL + && (technicalGrid.get(i+ii,j+jj,k+kk)->SOLVE & mask) == mask // Did that guy solve this component? + && technicalGrid.get(i+ii,j+jj,k+kk)->sysBoundaryFlag != sysboundarytype::DO_NOT_COMPUTE // Do not copy from there + ) { + distance = min(distance, ii*ii + jj*jj + kk*kk); + } + } + } } - #ifndef NDEBUG - - if ( technicalGrid.get(closestCell[0], closestCell[1], closestCell[2]) == nullptr ) { - stringstream ss; - ss << "ERROR, cell (" << gid[0] << "," << gid[1] << "," << gid[2] << ") tries to access invalid sysboundary nbr (" << ngid[0] << "," << ngid[1] << "," << ngid[2] << ") in " << __FILE__ << ":" << __LINE__ << endl; - cerr << ss.str(); - exit(1); + for (int kk=-2; kk<3; kk++) { + for (int jj=-2; jj<3; jj++) { + for (int ii=-2; ii<3 ; ii++) { + if( technicalGrid.get(i+ii,j+jj,k+kk) // skip invalid cells returning NULL + && (technicalGrid.get(i+ii,j+jj,k+kk)->SOLVE & mask) == mask // Did that guy solve this component? + && technicalGrid.get(i+ii,j+jj,k+kk)->sysBoundaryFlag != sysboundarytype::DO_NOT_COMPUTE // Do not copy from there + ) { + int d = ii*ii + jj*jj + kk*kk; + if( d == distance ) { + std::array cell = {i+ii, j+jj, k+kk}; + closestCells.push_back(cell); + } + } + } + } } - - if (technicalGrid.get(closestCell[0], closestCell[1], closestCell[2])->sysBoundaryFlag != sysboundarytype::NOT_SYSBOUNDARY) { - stringstream ss; - ss << "ERROR, cell (" << gid[0] << "," << gid[1] << "," << gid[2] << ") uses value from sysboundary nbr (" << ngid[0] << "," << ngid[1] << "," << ngid[2] << ") in " << __FILE__ << ":" << __LINE__ << endl; - cerr << ss.str(); - exit(1); + + if(closestCells.size() == 0) { + cerr << __FILE__ << ":" << __LINE__ << ": No closest cell found!" << endl; + abort(); } - - #endif - - return perBGrid.get(closestCell[0], closestCell[1], closestCell[2])->at(fsgrids::bfield::PERBX+component); + + return bGrid.get(closestCells[0][0], closestCells[0][1], closestCells[0][2])->at(fsgrids::bfield::PERBX+component); } /*! Function used in some cases to know which faces the system boundary condition is being applied to. diff --git a/sysboundary/sysboundarycondition.h b/sysboundary/sysboundarycondition.h index 355366031..4632d24cb 100644 --- a/sysboundary/sysboundarycondition.h +++ b/sysboundary/sysboundarycondition.h @@ -67,49 +67,45 @@ namespace SBC { )=0; virtual bool assignSysBoundary(dccrg::Dccrg& mpiGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid)=0; + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid)=0; virtual bool applyInitialState( const dccrg::Dccrg& mpiGrid, - FsGrid< std::array, 2> & perBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & perBGrid, Project &project )=0; virtual Real fieldSolverBoundaryCondMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< std::array, 2> & perBDt2Grid, - FsGrid< std::array, 2> & EGrid, - FsGrid< std::array, 2> & EDt2Grid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & bGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, creal& dt, - cuint& RKCase, cuint& component )=0; virtual void fieldSolverBoundaryCondElectricField( - FsGrid< std::array, 2> & EGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGrid, cint i, cint j, cint k, cuint component )=0; virtual void fieldSolverBoundaryCondHallElectricField( - FsGrid< std::array, 2> & EHallGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EHallGrid, cint i, cint j, cint k, cuint component )=0; virtual void fieldSolverBoundaryCondGradPeElectricField( - FsGrid< std::array, 2> & EGradPeGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & EGradPeGrid, cint i, cint j, cint k, cuint component )=0; virtual void fieldSolverBoundaryCondDerivatives( - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k, @@ -117,22 +113,22 @@ namespace SBC { cuint& component )=0; virtual void fieldSolverBoundaryCondBVOLDerivatives( - FsGrid< std::array, 2> & volGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, cint i, cint j, cint k, cuint& component )=0; static void setCellDerivativesToZero( - FsGrid< std::array, 2> & dPerBGrid, - FsGrid< std::array, 2> & dMomentsGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dPerBGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & dMomentsGrid, cint i, cint j, cint k, cuint& component ); static void setCellBVOLDerivativesToZero( - FsGrid< std::array, 2> & volGrid, + FsGrid< std::array, FS_STENCIL_WIDTH> & volGrid, cint i, cint j, cint k, @@ -252,13 +248,13 @@ namespace SBC { const uint popID ); std::array getTheClosestNonsysboundaryCell( - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k ); std::vector< std::array > getAllClosestNonsysboundaryCells( - FsGrid< fsgrids::technical, 2> & technicalGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k @@ -272,13 +268,14 @@ namespace SBC { std::vector & getAllCloseNonsysboundaryCells( const CellID& cellID ); - Real fieldBoundaryCopyFromExistingFaceNbrMagneticField( - FsGrid< std::array, 2> & perBGrid, - FsGrid< fsgrids::technical, 2> & technicalGrid, + Real fieldBoundaryCopyFromSolvingNbrMagneticField( + FsGrid< std::array, FS_STENCIL_WIDTH> & bGrid, + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, cint i, cint j, cint k, - cuint component + cuint component, + cuint mask ); /*! Precedence value of the system boundary condition. */ diff --git a/testpackage/run_tests.sh b/testpackage/run_tests.sh index 895eb4db8..be183faf3 100755 --- a/testpackage/run_tests.sh +++ b/testpackage/run_tests.sh @@ -135,7 +135,14 @@ do indices=(${variable_components[$run]// / }) for i in ${!variables[*]} do - if [ ! "${variables[$i]}" == "proton" ] + if [ "${variables[$i]}" == "fg_e" ] || [ "${variables[$i]}" == "fg_b" ] + then + relativeValue=$($run_command_tools vlsvdiff_DP --meshname=fsgrid ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The relative 0-distance between both datasets" |gawk '{print $8}' ) + absoluteValue=$($run_command_tools vlsvdiff_DP --meshname=fsgrid ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The absolute 0-distance between both datasets" |gawk '{print $8}' ) +#print the results + echo "${variables[$i]}_${indices[$i]} $absoluteValue $relativeValue " + + elif [ ! "${variables[$i]}" == "proton" ] then relativeValue=$($run_command_tools vlsvdiff_DP ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The relative 0-distance between both datasets" |gawk '{print $8}' ) absoluteValue=$($run_command_tools vlsvdiff_DP ${result_dir}/${comparison_vlsv[$run]} ${vlsv_dir}/${comparison_vlsv[$run]} ${variables[$i]} ${indices[$i]} |grep "The absolute 0-distance between both datasets" |gawk '{print $8}' ) diff --git a/testpackage/small_test_definitions.sh b/testpackage/small_test_definitions.sh index a925bc787..d6a4e5c7b 100644 --- a/testpackage/small_test_definitions.sh +++ b/testpackage/small_test_definitions.sh @@ -16,7 +16,7 @@ run_dir="run" test_dir="tests" # choose tests to run -run_tests=( 1 2 3 4 5 6 7 8 9 10 11 12 13 14) +run_tests=( 1 2 3 4 5 6 7 8 9 10 11 12 13 14 17) # acceleration test test_name[1]="acctest_2_maxw_500k_100k_20kms_10deg" @@ -122,17 +122,25 @@ comparison_phiprof[15]="phiprof_0.txt" variable_names[15]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e proton" variable_components[15]="0 0 1 2 0 1 2 0 1 2" -##AMR tests +## Velocity AMR tests # translation test test_name[16]="transtest_amr" -comparison_vlsv[3]="fullf.0000001.vlsv" -comparison_phiprof[3]="phiprof_0.txt" -variable_names[3]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton" -variable_components[3]="0 0 1 2" +comparison_vlsv[16]="fullf.0000001.vlsv" +comparison_phiprof[16]="phiprof_0.txt" +variable_names[16]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v proton" +variable_components[16]="0 0 1 2" +## Spatial AMR tests # Flowthrough test test_name[17]="Flowthrough_amr" -comparison_vlsv[12]="bulk.0000001.vlsv" -comparison_phiprof[12]="phiprof_0.txt" -variable_names[12]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" -variable_components[12]="0 0 1 2 0 1 2 0 1 2" +comparison_vlsv[17]="bulk.0000001.vlsv" +comparison_phiprof[17]="phiprof_0.txt" +variable_names[17]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[17]="0 0 1 2 0 1 2 0 1 2" + +# Magnetosphere 3D +test_name[18]="Magnetosphere_3D_small" +comparison_vlsv[18]="bulk.0000001.vlsv" +comparison_phiprof[18]="phiprof_0.txt" +variable_names[18]="proton/vg_rho proton/vg_v proton/vg_v proton/vg_v fg_b fg_b fg_b fg_e fg_e fg_e" +variable_components[18]="0 0 1 2 0 1 2 0 1 2" diff --git a/testpackage/tests/Flowthrough_amr/Flowthrough_amr.cfg b/testpackage/tests/Flowthrough_amr/Flowthrough_amr.cfg index 9d07e7394..f6f3c6736 100644 --- a/testpackage/tests/Flowthrough_amr/Flowthrough_amr.cfg +++ b/testpackage/tests/Flowthrough_amr/Flowthrough_amr.cfg @@ -18,11 +18,11 @@ box_half_width_z = 1 box_half_width_y = 1 [gridbuilder] -x_length = 14 +x_length = 16 y_length = 8 z_length = 8 -x_min = -7e7 -x_max = 7e7 +x_min = -8e7 +x_max = 8e7 y_min = -4e7 y_max = 4e7 z_min = -4e7 @@ -53,9 +53,11 @@ system_write_distribution_zline_stride = 0 [variables] output = populations_vg_rho +output = populations_vg_v output = fg_e output = fg_b output = vg_boundarytype +output = vg_boundarylayer output = vg_rank output = populations_vg_blocks diagnostic = populations_vg_blocks @@ -96,6 +98,9 @@ Bz = 1.0e-9 [proton_Flowthrough] T = 1.0e5 rho = 1.0e6 +VX0 = 1e5 +VY0 = 0 +VZ0 = 0 nSpaceSamples = 2 nVelocitySamples = 2 diff --git a/testpackage/tests/Flowthrough_amr/sw1.dat b/testpackage/tests/Flowthrough_amr/sw1.dat index 0a266607f..0496edeb9 100644 --- a/testpackage/tests/Flowthrough_amr/sw1.dat +++ b/testpackage/tests/Flowthrough_amr/sw1.dat @@ -1 +1 @@ -0.0 2.0e6 1.0e6 1.0e5 0.0 0.0 1.0e-9 1.0e-9 1.0e-9 +0.0 2.0e6 2.0e6 2.0e5 0.0 0.0 0.0 0.0 0.0 diff --git a/testpackage/tests/Magnetosphere_3D_small/Magnetosphere_3D_small.cfg b/testpackage/tests/Magnetosphere_3D_small/Magnetosphere_3D_small.cfg new file mode 100644 index 000000000..94bd518bd --- /dev/null +++ b/testpackage/tests/Magnetosphere_3D_small/Magnetosphere_3D_small.cfg @@ -0,0 +1,139 @@ +project = Magnetosphere +ParticlePopulations = proton +dynamic_timestep = 1 + +[proton_properties] +mass = 1 +mass_units = PROTON +charge = 1 + +[io] +diagnostic_write_interval = 1 +write_initial_state = 0 + +system_write_t_interval = 3 +system_write_file_name = bulk +system_write_distribution_stride = 0 +system_write_distribution_xline_stride = 10 +system_write_distribution_yline_stride = 10 +system_write_distribution_zline_stride = 1 + + +[gridbuilder] +x_length = 22 +y_length = 22 +z_length = 22 +x_min = -3.0e8 +x_max = 3.0e8 +y_min = -3.0e8 +y_max = 3.0e8 +z_min = -3.0e8 +z_max = 3.0e8 +t_max = 3.1 +#timestep_max = 100 + +[AMR] +max_spatial_level = 1 + +[proton_vspace] +vx_min = -4.02e6 +vx_max = +4.02e6 +vy_min = -4.02e6 +vy_max = +4.02e6 +vz_min = -4.02e6 +vz_max = +4.02e6 +vx_length = 67 +vy_length = 67 +vz_length = 67 +[proton_sparse] +minValue = 1.0e-15 + +[fieldsolver] +ohmHallTerm = 2 +minCFL = 0.4 +maxCFL = 0.5 + +[vlasovsolver] +minCFL = 0.8 +maxCFL = 0.99 +maxSlAccelerationRotation = 22 + +[loadBalance] +rebalanceInterval = 10 + +[variables] +output = populations_vg_rho +output = populations_vg_v +output = populations_vg_blocks +output = fg_b +output = fg_e +output = vg_pressure +output = vg_boundarytype +output = vg_boundarylayer +output = vg_rank +output = vg_f_saved +diagnostic = populations_vg_blocks + +[boundaries] +periodic_x = no +periodic_y = no +periodic_z = no +boundary = Outflow +boundary = Maxwellian +boundary = Ionosphere + +[ionosphere] +centerX = 0.0 +centerY = 0.0 +centerZ = 0.0 +radius = 70e6 +precedence = 2 + +[proton_ionosphere] +taperRadius = 1.5e8 +rho = 1.0e6 + +[outflow] +precedence = 3 +[proton_outflow] +face = x- +face = y- +face = y+ +face = z- +face = z+ + +[maxwellian] +face = x+ +precedence = 4 +[proton_maxwellian] +dynamic = 0 +file_x+ = sw1.dat + +[Magnetosphere] +constBgBX = 0.0 +constBgBY = 0.0 +constBgBZ = -5.0e-9 +noDipoleInSW = 1.0 + +dipoleType = 4 +dipoleTiltPhi = 0.0 +dipoleTiltTheta = 0 +dipoleXFull = 9.5565e7 # 15 RE +dipoleXZero = 2.5e8 +dipoleInflowBX = 0.0 +dipoleInflowBY = 0.0 +dipoleInflowBZ = 0.0 + +refine_L1radius = 1.2e8 +refine_L1tailthick = 4e7 + + +[proton_Magnetosphere] +T = 0.5e6 +rho = 1.0e6 +VX0 = -7.5e5 +VY0 = 0.0 +VZ0 = 0.0 +nSpaceSamples = 1 +nVelocitySamples = 1 + diff --git a/testpackage/tests/Magnetosphere_3D_small/sw1.dat b/testpackage/tests/Magnetosphere_3D_small/sw1.dat new file mode 100644 index 000000000..e8b278f28 --- /dev/null +++ b/testpackage/tests/Magnetosphere_3D_small/sw1.dat @@ -0,0 +1 @@ +0.0 1.0e6 0.5e6 -7.5e5 0.0 0.0 0.0e-9 0.0 0.0 diff --git a/testpackage/tests/restart_write/restart_write.cfg b/testpackage/tests/restart_write/restart_write.cfg index 857a08da8..6bd2e0b3b 100644 --- a/testpackage/tests/restart_write/restart_write.cfg +++ b/testpackage/tests/restart_write/restart_write.cfg @@ -7,6 +7,7 @@ dynamic_timestep = 1 ParticlePopulations = proton [io] +#write_as_float = 1 write_initial_state = 0 restart_walltime_interval = 1000 @@ -17,6 +18,10 @@ system_write_distribution_xline_stride = 0 system_write_distribution_yline_stride = 0 system_write_distribution_zline_stride = 0 +[restart] +#write_as_float = 1 + + [variables] output = vg_rhom output = fg_e diff --git a/tools/vlsvdiff.cpp b/tools/vlsvdiff.cpp index b7c1b276a..ef1eb7a5e 100644 --- a/tools/vlsvdiff.cpp +++ b/tools/vlsvdiff.cpp @@ -29,6 +29,8 @@ * Calling patterns are: * * "$ vlsvdiff ": Gives single-file statistics and distances between the two files given, for the variable and component given + + * "$ vlsvdiff --diff --meshname= ": Gives single-file statistics and distances between the two files given, for the variable and component given * * "$ vlsvdiff ": Gives single-file statistics and distances between pairs of files grid*.vlsv taken in alphanumeric order in the two folders given, for the variable and component given * @@ -38,6 +40,7 @@ #include +#include #include #include #include @@ -46,6 +49,7 @@ #include #include #include +#include #include #include #include @@ -64,6 +68,15 @@ using namespace vlsv; // equal to 'plaa'. static map attributes; +//Global enum and variable +static int gridName; +enum gridType{ + SpatialGrid, + fsgrid, + ionosphere +}; + + static uint64_t convUInt(const char* ptr, const vlsv::datatype::type& dataType, const uint64_t& dataSize) { if (dataType != vlsv::datatype::type::UINT) { cerr << "Erroneous datatype given to convUInt" << endl; @@ -111,8 +124,9 @@ bool copyArray(vlsv::Reader& input,vlsv::Writer& output, // Figure out arraysize, vectorsize, datasize, and datatype of the copied array map::const_iterator it; + map::iterator it2; it = outputAttribs.find("arraysize"); if (it == outputAttribs.end()) return false; - const uint64_t arraysize = atol(it->second.c_str()); + uint64_t arraysize = atol(it->second.c_str()); it = outputAttribs.find("vectorsize"); if (it == outputAttribs.end()) return false; const uint64_t vectorsize = atol(it->second.c_str()); it = outputAttribs.find("datasize"); if (it == outputAttribs.end()) return false; @@ -129,6 +143,7 @@ bool copyArray(vlsv::Reader& input,vlsv::Writer& output, delete [] ptr; return false; } + // Write array to output file if (output.writeArray(tagName,outputAttribs,datatype,arraysize,vectorsize,datasize,ptr) == false) { cerr << "ERROR: Failed to write array '" << tagName << "' in " << __FILE__ << ":" << __LINE__ << endl; @@ -139,12 +154,127 @@ bool copyArray(vlsv::Reader& input,vlsv::Writer& output, return success; } + +/* Small function that overrides how fsgrid diff files are written*/ +bool HandleFsGrid(const string& inputFileName, + vlsv::Writer& output, + std::map orderedData) +{ + + + //Open input file + vlsv::Reader input; + if (input.open(inputFileName) == false) { + cerr << "ERROR failed to open input file '" << inputFileName << "' in " << __FILE__ << ":" << __LINE__ << endl; + return false; + } + + //Read Mesh Attributes + std::string tagName="MESH"; + list > inputAttribs; + inputAttribs.push_back(make_pair("name","fsgrid")); + map outputAttribs; + + if (input.getArrayAttributes(tagName,inputAttribs,outputAttribs) == false) { + cerr << "ERROR: Failed to read array '" << tagName << "' attributes in " << __FILE__ << ":" << __LINE__ << endl; + cerr << "Input attributes are:" << endl; + for (list >::const_iterator it=inputAttribs.begin(); it!=inputAttribs.end(); ++it) { + cerr << "\t '" << it->first << "' = '" << it->second << "'" << endl; + } + return false; + } + + //Collect needed attributes to a map named patch + map::const_iterator it; + it = outputAttribs.find("arraysize"); if (it == outputAttribs.end()) return false; + uint64_t arraysize = atol(it->second.c_str()); + it = outputAttribs.find("vectorsize"); if (it == outputAttribs.end()) return false; + const uint64_t vectorsize = atol(it->second.c_str()); + it = outputAttribs.find("datasize"); if (it == outputAttribs.end()) return false; + const uint64_t datasize = atol(it->second.c_str()); + it = outputAttribs.find("datatype"); if (it == outputAttribs.end()) return false; + const string datatype = it->second; + it = outputAttribs.find("xperiodic"); if (it == outputAttribs.end()) return false; + const string xperiodic = it->second; + it = outputAttribs.find("yperiodic"); if (it == outputAttribs.end()) return false; + const string yperiodic = it->second; + it = outputAttribs.find("zperiodic"); if (it == outputAttribs.end()) return false; + const string zperiodic = it->second; + it = outputAttribs.find("type"); if (it == outputAttribs.end()) return false; + const string type = it->second; + + mappatch; + patch["arraysize"]=std::to_string(arraysize); + patch["datasize"]=std::to_string(datasize); + patch["datatype"]=datatype; + patch["name"]="fsgrid"; + patch["type"]=type; + patch["vectorsize"]=std::to_string(vectorsize); + patch["xperiodic"]=xperiodic; + patch["yperiodic"]=yperiodic; + patch["zperiodic"]=zperiodic; + + + //Get the global IDs in a vector + std::vector globalIds; + for (const auto iter : orderedData){ + globalIds.push_back( iter.first ); + } + + //Write to file + output.writeArray("MESH",patch,arraysize,1,&globalIds[0]); + + + //Now for MESH_DOMAIN_SIZES + inputAttribs.clear(); + inputAttribs.push_back(make_pair("mesh","fsgrid")); + tagName="MESH_DOMAIN_SIZES"; + + if (input.getArrayAttributes(tagName,inputAttribs,outputAttribs) == false) { + cerr << "ERROR: Failed to read array '" << tagName << "' attributes in " << __FILE__ << ":" << __LINE__ << endl; + cerr << "Input attributes are:" << endl; + for (list >::const_iterator it=inputAttribs.begin(); it!=inputAttribs.end(); ++it) { + cerr << "\t '" << it->first << "' = '" << it->second << "'" << endl; + } + return false; + } + + + //Read some attributes we need and parse to our map + it = outputAttribs.find("datasize"); if (it == outputAttribs.end()) return false; + const uint64_t datasize2 = atol(it->second.c_str()); + it = outputAttribs.find("datatype"); if (it == outputAttribs.end()) return false; + const string datatype2 = it->second; + it = outputAttribs.find("vectorsize"); if (it == outputAttribs.end()) return false; + const uint64_t vectorsize2 = atol(it->second.c_str()); + + patch.clear(); + patch["arraysize"]="1"; + patch["datasize"]=to_string(datasize2); + patch["datatype"]=datatype2; + patch["mesh"]="fsgrid"; + patch["vectorsize"]=to_string(vectorsize2); + + //Override MESH_DOMAIN_SIZES + std::array meshDomainSize({globalIds.size(), 0}); + output.writeArray("MESH_DOMAIN_SIZES",patch ,1,vectorsize2, &meshDomainSize[0]); + + + //Close the file + input.close(); + + + return true; + + +} + /** Copy the spatial mesh from input to output. * @param inputFileName Name of the input file where the mesh is copied from. * @param output VLSV reader for the file where the cloned mesh is written. * @param meshName Name of the mesh. * @return If true, the mesh was successfully cloned.*/ -bool cloneMesh(const string& inputFileName,vlsv::Writer& output,const string& meshName) { +bool cloneMesh(const string& inputFileName,vlsv::Writer& output,const string& meshName, std::map orderedData) { bool success = true; vlsv::Reader input; @@ -155,22 +285,105 @@ bool cloneMesh(const string& inputFileName,vlsv::Writer& output,const string& me list > inputAttribs; inputAttribs.push_back(make_pair("name",meshName)); - if (copyArray(input,output,"MESH",inputAttribs) == false) success = false; - inputAttribs.clear(); inputAttribs.push_back(make_pair("mesh",meshName)); if (copyArray(input,output,"MESH_BBOX",inputAttribs) == false) success = false; - if (copyArray(input,output,"MESH_DOMAIN_SIZES",inputAttribs) == false) success = false; if (copyArray(input,output,"MESH_NODE_CRDS_X",inputAttribs) == false) success = false; if (copyArray(input,output,"MESH_NODE_CRDS_Y",inputAttribs) == false) success = false; if (copyArray(input,output,"MESH_NODE_CRDS_Z",inputAttribs) == false) success = false; if (copyArray(input,output,"MESH_GHOST_LOCALIDS",inputAttribs) == false) success = false; if (copyArray(input,output,"MESH_GHOST_DOMAINS",inputAttribs) == false) success = false; + + //Only do this if we diff SpatialGrid data + if (gridName==gridType::SpatialGrid){ + if (copyArray(input,output,"MESH_DOMAIN_SIZES",inputAttribs) == false) success = false; + + inputAttribs.clear(); + inputAttribs.push_back(make_pair("name",meshName)); + if (copyArray(input,output,"MESH",inputAttribs) == false) success = false; + }else{ + HandleFsGrid(inputFileName,output,orderedData); + } input.close(); return success; } + //! Helper function: calculate position of the local coordinate space for the given dimension + // \param globalCells Number of cells in the global Simulation, in this dimension + // \param ntasks Total number of tasks in this dimension + // \param my_n This task's position in this dimension + // \return Cell number at which this task's domains cells start (actual cells, not counting ghost cells) + int32_t calcLocalStart(int32_t globalCells, int ntasks, int my_n) { + int n_per_task = globalCells / ntasks; + int remainder = globalCells % ntasks; + + if(my_n < remainder) { + return my_n * (n_per_task+1); + } else { + return my_n * n_per_task + remainder; + } + } + //! Helper function: calculate size of the local coordinate space for the given dimension + // \param globalCells Number of cells in the global Simulation, in this dimension + // \param ntasks Total number of tasks in this dimension + // \param my_n This task's position in this dimension + // \return Nmuber of cells for this task's local domain (actual cells, not counting ghost cells) + int32_t calcLocalSize(int32_t globalCells, int ntasks, int my_n) { + int n_per_task = globalCells/ntasks; + int remainder = globalCells%ntasks; + if(my_n < remainder) { + return n_per_task+1; + } else { + return n_per_task; + } + } + + //! Helper function to optimize decomposition of this grid over the given number of tasks + void computeDomainDecomposition(const std::array& GlobalSize, int nProcs, std::array& processDomainDecomposition) { + std::array systemDim; + std::array processBox; + double optimValue = std::numeric_limits::max(); + for(int i = 0; i < 3; i++) { + systemDim[i] = (double)GlobalSize[i]; + } + processDomainDecomposition = {1, 1, 1}; + for (int i = 1; i <= std::min(nProcs, GlobalSize[0]); i++) { + processBox[0] = std::max(systemDim[0]/i, 1.0); + for (int j = 1; j <= std::min(nProcs, GlobalSize[1]) ; j++) { + if( i * j > nProcs ) + break; + processBox[1] = std::max(systemDim[1]/j, 1.0); + for (int k = 1; k <= std::min(nProcs, GlobalSize[2]); k++) { + if( i * j * k > nProcs ) + break; + processBox[2] = std::max(systemDim[2]/k, 1.0); + double value = + 10 * processBox[0] * processBox[1] * processBox[2] + + (i > 1 ? processBox[1] * processBox[2]: 0) + + (j > 1 ? processBox[0] * processBox[2]: 0) + + (k > 1 ? processBox[0] * processBox[1]: 0); + + if(value < optimValue ){ + optimValue = value; + processDomainDecomposition[0] = i; + processDomainDecomposition[1] = j; + processDomainDecomposition[2] = k; + } + } + } + } + + if(optimValue == std::numeric_limits::max() || + processDomainDecomposition[0] * processDomainDecomposition[1] * processDomainDecomposition[2] != nProcs) { + std::cerr << "FSGrid domain decomposition failed, are you running on a prime number of tasks?" << std::endl; + throw std::runtime_error("FSGrid computeDomainDecomposition failed"); + } + } + + + + /*! Extracts the dataset from the VLSV file opened by convertSILO. * \param vlsvReader vlsvinterface::Reader class object used to access the VLSV file * \param meshName Address of the string containing the name of the mesh to be extracted @@ -208,71 +421,168 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader, cerr << "ERROR, failed to get array info for '" << _varToExtract << "' at " << __FILE__ << " " << __LINE__ << endl; return false; } + char *variableBuffer = new char[variableVectorSize * variableDataSize]; + float *variablePtrFloat = reinterpret_cast(variableBuffer); + double *variablePtrDouble = reinterpret_cast(variableBuffer); + uint *variablePtrUint = reinterpret_cast(variableBuffer); + int *variablePtrInt = reinterpret_cast(variableBuffer); + + + if (gridName==gridType::SpatialGrid){ + + // Read the mesh array one node (of a spatial cell) at a time + // and create a map which contains each cell's CellID and variable to be extracted + //Get local cell ids: + vector local_cells; + if ( vlsvReader.getCellIds( local_cells, meshName) == false ) { + cerr << "Failed to read cell ids at " << __FILE__ << " " << __LINE__ << endl; + return false; + } - //Get local cell ids: - vector local_cells; - if ( vlsvReader.getCellIds( local_cells, meshName) == false ) { - cerr << "Failed to read cell ids at " << __FILE__ << " " << __LINE__ << endl; - return false; - } + //Check for correct output: + if (local_cells.size() != variableArraySize) { + cerr << "ERROR array size mismatch: " << local_cells.size() << " " << variableArraySize << endl; + } + if (compToExtract + 1 > variableVectorSize) { + cerr << "ERROR invalid component, this variable has size " << variableVectorSize << endl; + abort(); + } + + if (storeCellOrder == true) { + cellOrder.clear(); + } + + orderedData->clear(); + + for (uint64_t i=0; iinsert(pair(CellID, extract)); + if (storeCellOrder == true) { + cellOrder[CellID] = i; + } + } + + }else if (gridName==gridType::fsgrid){ + + + int numtasks; + int xcells,ycells,zcells; + vlsvReader.readParameter("numWritingRanks",numtasks); + vlsvReader.readParameter("xcells_ini",xcells); + vlsvReader.readParameter("ycells_ini",ycells); + vlsvReader.readParameter("zcells_ini",zcells); + std::array GlobalBox={xcells,ycells,zcells}; + std::array thisDomainDecomp; + + //Compute Domain Decomposition Scheme for this vlsv file + computeDomainDecomposition(GlobalBox,numtasks,thisDomainDecomp); - //Check for correct output: - if (local_cells.size() != variableArraySize) { - cerr << "ERROR array size mismatch: " << local_cells.size() << " " << variableArraySize << endl; - } - if (compToExtract + 1 > variableVectorSize) { - cerr << "ERROR invalid component, this variable has size " << variableVectorSize << endl; - abort(); - } - - // Read the mesh array one node (of a spatial cell) at a time - // and create a map which contains each cell's CellID and variable to be extracted - char* variableBuffer = new char[variableVectorSize*variableDataSize]; - float* variablePtrFloat = reinterpret_cast(variableBuffer); - double* variablePtrDouble = reinterpret_cast(variableBuffer); - uint* variablePtrUint = reinterpret_cast(variableBuffer); - int* variablePtrInt = reinterpret_cast(variableBuffer); - if (storeCellOrder == true) { - cellOrder.clear(); - } - - orderedData->clear(); + std::array taskSize,taskStart; + std::array taskEnd; + int readOffset=0; + size_t readSize; + int index,my_x,my_y,my_z; + orderedData->clear(); + + //Read into buffer + for (int task=0; task readIn(variableVectorSize * variableDataSize*readSize); - switch (variableDataType) { - case datatype::type::FLOAT: - if(variableDataSize == sizeof(float)) extract = (Real)(variablePtrFloat[compToExtract]); - if(variableDataSize == sizeof(double)) extract = (Real)(variablePtrDouble[compToExtract]); - break; - case datatype::type::UINT: - extract = (Real)(variablePtrUint[compToExtract]); - break; - case datatype::type::INT: - extract = (Real)(variablePtrInt[compToExtract]); - break; - case datatype::type::UNKNOWN: - cerr << "ERROR, BAD DATATYPE AT " << __FILE__ << " " << __LINE__ << endl; - break; - } - // Put those into the map - orderedData->insert(pair(CellID, extract)); - if (storeCellOrder == true) { - cellOrder[CellID] = i; + + int counter2=0; + uint64_t globalindex; + int64_t counter=0; + for(int z=taskStart[2]; zinsert(pair(globalindex, extract)); + counter++; + + } + } + } + readOffset+=readSize; } + }else{ + cerr<<"meshName not recognized\t" << __FILE__ << " " << __LINE__ <& orderedData1, value = abs(it1->second - it2->second); *absolute = max(*absolute, value); length = max(length, abs(it1->second)); - } - array[cellOrder.at(it1->first)] = value; + + } + if (gridName==gridType::SpatialGrid){ + array[cellOrder.at(it1->first)] = value; + }else if (gridName==gridType::fsgrid) { + + array.at(it1->first)=value; + } } } else if (p == 1) { for (map::const_iterator it1=orderedData1.begin(); it1!=orderedData1.end(); ++it1) { @@ -437,8 +754,13 @@ bool pDistance(const map& orderedData1, value = abs(it1->second - it2->second); *absolute += value; length += abs(it1->second); - } - array[cellOrder.at(it1->first)] = value; + + } + if (gridName==gridType::SpatialGrid){ + array[cellOrder.at(it1->first)] = value; + }else if (gridName==gridType::fsgrid){ + array[it1->first]=value; + } } } else { for (map::const_iterator it1=orderedData1.begin(); it1!=orderedData1.end(); ++it1) { @@ -448,8 +770,13 @@ bool pDistance(const map& orderedData1, value = pow(abs(it1->second - it2->second), p); *absolute += value; length += pow(abs(it1->second), p); - } - array[cellOrder.at(it1->first)] = pow(value,1.0/p); + + } + if (gridName==gridType::SpatialGrid){ + array[cellOrder.at(it1->first)] = pow(value,1.0/p); + }else if (gridName==gridType::fsgrid){ + array[it1->first]=pow(value,1.0/p); + } } *absolute = pow(*absolute, 1.0 / p); length = pow(length, 1.0 / p); @@ -466,6 +793,8 @@ bool pDistance(const map& orderedData1, map attributes; attributes["mesh"] = meshName; attributes["name"] = varName; + + if (outputFile.writeArray("VARIABLE",attributes,array.size(),1,&(array[0])) == false) { cerr << "ERROR failed to write variable '" << varName << "' to output file in " << __FILE__ << ":" << __LINE__ << endl; return 1; @@ -1216,12 +1545,16 @@ bool process2Files(const string fileName1, return false; } - // Clone mesh from input file to diff file + map::const_iterator it = attributes.find("--meshname"); - if (cloneMesh(fileName1,outputFile,it->second) == false) return false; + if (cloneMesh(fileName1,outputFile,it->second,orderedData1) == false) { + std::cerr<<"Failed"<::const_iterator it=defAttribs.begin(); it!=defAttribs.end(); ++it) { if (it->second.size() == 0) continue; @@ -1418,6 +1752,8 @@ int main(int argn,char* args[]) { return 0; } + + if (argsVector.size() < 5) { cout << endl; cout << "USAGE 1: ./vlsvdiff " << endl; @@ -1450,6 +1786,23 @@ int main(int argn,char* args[]) { compToExtract2 = compToExtract; } + + //Figure out Meshname + if (attributes["--meshname"] == "SpatialGrid") { + gridName=gridType::SpatialGrid ; + }else if (attributes["--meshname"]=="fsgrid"){ + gridName=gridType::fsgrid ; + }else if (attributes["--meshname"]=="ionosphere"){ + gridName=gridType::ionosphere ; + }else{ + std::cout<& mpiGrid, - FsGrid< fsgrids::technical, 2>& technicalGrid, Real &newDt, bool &isChanged) { + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technicalGrid, Real &newDt, bool &isChanged) { phiprof::start("compute-timestep"); //compute maximum time-step, this cannot be done at the first @@ -401,19 +401,19 @@ int main(int argn,char* args[]) { sysBoundaries.isBoundaryPeriodic(2)}; FsGridCouplingInformation gridCoupling; - FsGrid< std::array, 2> perBGrid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< std::array, 2> perBDt2Grid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< std::array, 2> EGrid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< std::array, 2> EDt2Grid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< std::array, 2> EHallGrid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< std::array, 2> EGradPeGrid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< std::array, 2> momentsGrid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< std::array, 2> momentsDt2Grid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< std::array, 2> dPerBGrid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< std::array, 2> dMomentsGrid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< std::array, 2> BgBGrid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< std::array, 2> volGrid(fsGridDimensions, comm, periodicity,gridCoupling); - FsGrid< fsgrids::technical, 2> technicalGrid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> perBGrid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> perBDt2Grid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> EGrid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> EDt2Grid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> EHallGrid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> EGradPeGrid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> momentsGrid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> momentsDt2Grid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> dPerBGrid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> dMomentsGrid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> BgBGrid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< std::array, FS_STENCIL_WIDTH> volGrid(fsGridDimensions, comm, periodicity,gridCoupling); + FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> technicalGrid(fsGridDimensions, comm, periodicity,gridCoupling); // Set DX,DY and DZ // TODO: This is currently just taking the values from cell 1, and assuming them to be @@ -433,6 +433,13 @@ int main(int argn,char* args[]) { = momentsDt2Grid.physicalGlobalStart = dPerBGrid.physicalGlobalStart = dMomentsGrid.physicalGlobalStart = BgBGrid.physicalGlobalStart = volGrid.physicalGlobalStart = technicalGrid.physicalGlobalStart = {P::xmin, P::ymin, P::zmin}; + + // Checking that spatial cells are cubic, otherwise field solver is incorrect (cf. derivatives in E, Hall term) + if((abs((technicalGrid.DX-technicalGrid.DY)/technicalGrid.DX) > 0.001) || + (abs((technicalGrid.DX-technicalGrid.DZ)/technicalGrid.DX) > 0.001) || + (abs((technicalGrid.DY-technicalGrid.DZ)/technicalGrid.DY) > 0.001)) { + std::cerr << "WARNING: Your spatial cells seem not to be cubic. However the field solver is assuming them to be. Use at your own risk and responsibility!" << std::endl; + } phiprof::stop("Init fieldsolver grids"); // Initialize grid. After initializeGrid local cells have dist @@ -535,7 +542,7 @@ int main(int argn,char* args[]) { BgBGrid, volGrid, technicalGrid, - &outputReducer,P::systemWriteName.size()-1, writeGhosts) == false ) { + &outputReducer,P::systemWriteName.size()-1, P::restartStripeFactor, writeGhosts) == false ) { cerr << "FAILED TO WRITE GRID AT " << __FILE__ << " " << __LINE__ << endl; } @@ -595,8 +602,12 @@ int main(int argn,char* args[]) { //is requested for writing, then jump to next writing index. This is to //make sure that at restart we do not write in the middle of //the interval. - if(P::t_min>(index+0.01)*P::systemWriteTimeInterval[i]) + if(P::t_min>(index+0.01)*P::systemWriteTimeInterval[i]) { index++; + // Special case for large timesteps + int index2=(int)((P::t_min+P::dt)/P::systemWriteTimeInterval[i]); + if (index2>index) index=index2; + } P::systemWrites.push_back(index); } @@ -676,6 +687,14 @@ int main(int argn,char* args[]) { for (uint i = 0; i < P::systemWriteTimeInterval.size(); i++) { if (P::systemWriteTimeInterval[i] >= 0.0 && P::t >= P::systemWrites[i] * P::systemWriteTimeInterval[i] - DT_EPSILON) { + // If we have only just restarted, the bulk file should already exist from the previous slot. + if ((P::tstep == P::tstep_min) && (P::tstep>0)) { + P::systemWrites[i]++; + // Special case for large timesteps + int index2=(int)((P::t+P::dt)/P::systemWriteTimeInterval[i]); + if (index2>P::systemWrites[i]) P::systemWrites[i]=index2; + continue; + } phiprof::start("write-system"); logFile << "(IO): Writing spatial cell and reduced system data to disk, tstep = " << P::tstep << " t = " << P::t << endl << writeVerbose; @@ -691,10 +710,13 @@ int main(int argn,char* args[]) { BgBGrid, volGrid, technicalGrid, - &outputReducer, i, writeGhosts) == false ) { + &outputReducer, i, P::bulkStripeFactor, writeGhosts) == false ) { cerr << "FAILED TO WRITE GRID AT" << __FILE__ << " " << __LINE__ << endl; } P::systemWrites[i]++; + // Special case for large timesteps + int index2=(int)((P::t+P::dt)/P::systemWriteTimeInterval[i]); + if (index2>P::systemWrites[i]) P::systemWrites[i]=index2; logFile << "(IO): .... done!" << endl << writeVerbose; phiprof::stop("write-system"); } @@ -759,7 +781,7 @@ int main(int argn,char* args[]) { BgBGrid, volGrid, technicalGrid, - outputReducer,"restart",(uint)P::t, P::restartStripeFactor) == false ) { + outputReducer,"restart",(uint)P::t,P::restartStripeFactor) == false ) { logFile << "(IO): ERROR Failed to write restart!" << endl << writeVerbose; cerr << "FAILED TO WRITE RESTART" << endl; } @@ -887,8 +909,8 @@ int main(int argn,char* args[]) { phiprof::start("fsgrid-coupling-in"); // Copy moments over into the fsgrid. //setupTechnicalFsGrid(mpiGrid, cells, technicalGrid); - feedMomentsIntoFsGrid(mpiGrid, cells, momentsGrid,false); - feedMomentsIntoFsGrid(mpiGrid, cells, momentsDt2Grid,true); + feedMomentsIntoFsGrid(mpiGrid, cells, momentsGrid, technicalGrid, false); + feedMomentsIntoFsGrid(mpiGrid, cells, momentsDt2Grid, technicalGrid, true); phiprof::stop("fsgrid-coupling-in"); propagateFields( diff --git a/vlasovsolver/cpu_1d_plm.hpp b/vlasovsolver/cpu_1d_plm.hpp index fe12ec89c..33194c261 100644 --- a/vlasovsolver/cpu_1d_plm.hpp +++ b/vlasovsolver/cpu_1d_plm.hpp @@ -39,11 +39,17 @@ The factor 2.0 is in the polynom to ease integration, then integral is a[0]*t + */ inline void compute_plm_coeff(const Vec * const values, uint k, Vec a[2], const Realv threshold){ - // scale values closer to 1 for more accurate slope limiter calculation - const Realv scale = 1./threshold; - const Vec d_cv=slope_limiter(values[k - 1]*scale, values[k]*scale, values[k + 1]*scale)*threshold; - a[0] = values[k] - d_cv * 0.5; - a[1] = d_cv * 0.5; + if (threshold>0) { + // scale values closer to 1 for more accurate slope limiter calculation + const Realv scale = 1./threshold; + const Vec d_cv=slope_limiter(values[k - 1]*scale, values[k]*scale, values[k + 1]*scale)*threshold; + a[0] = values[k] - d_cv * 0.5; + a[1] = d_cv * 0.5; + } else { + const Vec d_cv=slope_limiter(values[k - 1], values[k], values[k + 1]); + a[0] = values[k] - d_cv * 0.5; + a[1] = d_cv * 0.5; + } } #endif diff --git a/vlasovsolver/cpu_acc_load_blocks.cpp b/vlasovsolver/cpu_acc_load_blocks.cpp index 0a1bcacec..05ad92215 100644 --- a/vlasovsolver/cpu_acc_load_blocks.cpp +++ b/vlasovsolver/cpu_acc_load_blocks.cpp @@ -127,6 +127,26 @@ for dimension in range(0, 2): values[i_pcolumnv_b(2, 3, block_k, n_blocks)] = gather4d<11 ,27 ,43 ,59>(data); values[i_pcolumnv_b(3, 3, block_k, n_blocks)] = gather4d<15 ,31 ,47 ,63>(data); #endif //VEC4D_AGNER + +#if defined(VEC4F_FALLBACK) || defined(VEC4D_FALLBACK) + values[i_pcolumnv_b(0, 0, block_k, n_blocks)] = Vec(data[0], data[16], data[32], data[48]); + values[i_pcolumnv_b(1, 0, block_k, n_blocks)] = Vec(data[4], data[20], data[36], data[52]); + values[i_pcolumnv_b(2, 0, block_k, n_blocks)] = Vec(data[8], data[24], data[40], data[56]); + values[i_pcolumnv_b(3, 0, block_k, n_blocks)] = Vec(data[12], data[28], data[44], data[60]); + values[i_pcolumnv_b(0, 1, block_k, n_blocks)] = Vec(data[1], data[17], data[33], data[49]); + values[i_pcolumnv_b(1, 1, block_k, n_blocks)] = Vec(data[5], data[21], data[37], data[53]); + values[i_pcolumnv_b(2, 1, block_k, n_blocks)] = Vec(data[9], data[25], data[41], data[57]); + values[i_pcolumnv_b(3, 1, block_k, n_blocks)] = Vec(data[13], data[29], data[45], data[61]); + values[i_pcolumnv_b(0, 2, block_k, n_blocks)] = Vec(data[2], data[18], data[34], data[50]); + values[i_pcolumnv_b(1, 2, block_k, n_blocks)] = Vec(data[6], data[22], data[38], data[54]); + values[i_pcolumnv_b(2, 2, block_k, n_blocks)] = Vec(data[10], data[26], data[42], data[58]); + values[i_pcolumnv_b(3, 2, block_k, n_blocks)] = Vec(data[14], data[30], data[46], data[62]); + values[i_pcolumnv_b(0, 3, block_k, n_blocks)] = Vec(data[3], data[19], data[35], data[51]); + values[i_pcolumnv_b(1, 3, block_k, n_blocks)] = Vec(data[7], data[23], data[39], data[55]); + values[i_pcolumnv_b(2, 3, block_k, n_blocks)] = Vec(data[11], data[27], data[43], data[59]); + values[i_pcolumnv_b(3, 3, block_k, n_blocks)] = Vec(data[15], data[31], data[47], data[63]); +#endif //VEC4F_FALLBACK || VEC4D_FALLBACK + #ifdef VEC8F_AGNER values[i_pcolumnv_b(0, 0, block_k, n_blocks)] = gather8f<0 ,16 ,32 ,48 ,4 ,20 ,36 ,52>(data); values[i_pcolumnv_b(1, 0, block_k, n_blocks)] = gather8f<8 ,24 ,40 ,56 ,12 ,28 ,44 ,60>(data); @@ -147,6 +167,18 @@ for dimension in range(0, 2): values[i_pcolumnv_b(0, 3, block_k, n_blocks)] = gather8d<3 ,19 ,35 ,51 ,7 ,23 ,39 ,55>(data); values[i_pcolumnv_b(1, 3, block_k, n_blocks)] = gather8d<11 ,27 ,43 ,59 ,15 ,31 ,47 ,63>(data); #endif //VEC8D_AGNER + +#if defined(VEC8F_FALLBACK) || defined(VEC8D_FALLBACK) + values[i_pcolumnv_b(0, 0, block_k, n_blocks)] = Vec(data[0], data[16], data[32], data[48], data[4], data[20], data[36], data[52]); + values[i_pcolumnv_b(1, 0, block_k, n_blocks)] = Vec(data[8], data[24], data[40], data[56], data[12], data[28], data[44], data[60]); + values[i_pcolumnv_b(0, 1, block_k, n_blocks)] = Vec(data[1], data[17], data[33], data[49], data[5], data[21], data[37], data[53]); + values[i_pcolumnv_b(1, 1, block_k, n_blocks)] = Vec(data[9], data[25], data[41], data[57], data[13], data[29], data[45], data[61]); + values[i_pcolumnv_b(0, 2, block_k, n_blocks)] = Vec(data[2], data[18], data[34], data[50], data[6], data[22], data[38], data[54]); + values[i_pcolumnv_b(1, 2, block_k, n_blocks)] = Vec(data[10], data[26], data[42], data[58], data[14], data[30], data[46], data[62]); + values[i_pcolumnv_b(0, 3, block_k, n_blocks)] = Vec(data[3], data[19], data[35], data[51], data[7], data[23], data[39], data[55]); + values[i_pcolumnv_b(1, 3, block_k, n_blocks)] = Vec(data[11], data[27], data[43], data[59], data[15], data[31], data[47], data[63]); +#endif //VEC8F_FALLBACK || VEC8D_FALLBACK + #ifdef VEC16F_AGNER values[i_pcolumnv_b(0, 0, block_k, n_blocks)] = gather16f<0 ,16 ,32 ,48 ,4 ,20 ,36 ,52 ,8 ,24 ,40 ,56 ,12 ,28 ,44 ,60>(data); values[i_pcolumnv_b(0, 1, block_k, n_blocks)] = gather16f<1 ,17 ,33 ,49 ,5 ,21 ,37 ,53 ,9 ,25 ,41 ,57 ,13 ,29 ,45 ,61>(data); @@ -186,6 +218,26 @@ for dimension in range(0, 2): values[i_pcolumnv_b(2, 3, block_k, n_blocks)] = gather4f<44 ,45 ,46 ,47>(data); values[i_pcolumnv_b(3, 3, block_k, n_blocks)] = gather4f<60 ,61 ,62 ,63>(data); #endif //VEC4F_AGNER + +#if defined(VEC4F_FALLBACK) || defined(VEC4D_FALLBACK) + values[i_pcolumnv_b(0, 0, block_k, n_blocks)] = Vec(data[0], data[1], data[2], data[3]); + values[i_pcolumnv_b(1, 0, block_k, n_blocks)] = Vec(data[16], data[17], data[18], data[19]); + values[i_pcolumnv_b(2, 0, block_k, n_blocks)] = Vec(data[32], data[33], data[34], data[35]); + values[i_pcolumnv_b(3, 0, block_k, n_blocks)] = Vec(data[48], data[49], data[50], data[51]); + values[i_pcolumnv_b(0, 1, block_k, n_blocks)] = Vec(data[4], data[5], data[6], data[7]); + values[i_pcolumnv_b(1, 1, block_k, n_blocks)] = Vec(data[20], data[21], data[22], data[23]); + values[i_pcolumnv_b(2, 1, block_k, n_blocks)] = Vec(data[36], data[37], data[38], data[39]); + values[i_pcolumnv_b(3, 1, block_k, n_blocks)] = Vec(data[52], data[53], data[54], data[55]); + values[i_pcolumnv_b(0, 2, block_k, n_blocks)] = Vec(data[8], data[9], data[10], data[11]); + values[i_pcolumnv_b(1, 2, block_k, n_blocks)] = Vec(data[24], data[25], data[26], data[27]); + values[i_pcolumnv_b(2, 2, block_k, n_blocks)] = Vec(data[40], data[41], data[42], data[43]); + values[i_pcolumnv_b(3, 2, block_k, n_blocks)] = Vec(data[56], data[57], data[58], data[59]); + values[i_pcolumnv_b(0, 3, block_k, n_blocks)] = Vec(data[12], data[13], data[14], data[15]); + values[i_pcolumnv_b(1, 3, block_k, n_blocks)] = Vec(data[28], data[29], data[30], data[31]); + values[i_pcolumnv_b(2, 3, block_k, n_blocks)] = Vec(data[44], data[45], data[46], data[47]); + values[i_pcolumnv_b(3, 3, block_k, n_blocks)] = Vec(data[60], data[61], data[62], data[63]); +#endif //VEC4F_FALLBACK || VEC4D_FALLBACK + #ifdef VEC4D_AGNER values[i_pcolumnv_b(0, 0, block_k, n_blocks)] = gather4d<0 ,1 ,2 ,3>(data); values[i_pcolumnv_b(1, 0, block_k, n_blocks)] = gather4d<16 ,17 ,18 ,19>(data); @@ -224,6 +276,18 @@ for dimension in range(0, 2): values[i_pcolumnv_b(0, 3, block_k, n_blocks)] = gather8d<12 ,13 ,14 ,15 ,28 ,29 ,30 ,31>(data); values[i_pcolumnv_b(1, 3, block_k, n_blocks)] = gather8d<44 ,45 ,46 ,47 ,60 ,61 ,62 ,63>(data); #endif //VEC8D_AGNER + +#if defined (VEC8F_FALLBACK) || defined (VEC8D_FALLBACK) + values[i_pcolumnv_b(0, 0, block_k, n_blocks)] = Vec(data[0], data[1], data[2], data[3], data[16], data[17], data[18], data[19]); + values[i_pcolumnv_b(1, 0, block_k, n_blocks)] = Vec(data[32], data[33], data[34], data[35], data[48], data[49], data[50], data[51]); + values[i_pcolumnv_b(0, 1, block_k, n_blocks)] = Vec(data[4], data[5], data[6], data[7], data[20], data[21], data[22], data[23]); + values[i_pcolumnv_b(1, 1, block_k, n_blocks)] = Vec(data[36], data[37], data[38], data[39], data[52], data[53], data[54], data[55]); + values[i_pcolumnv_b(0, 2, block_k, n_blocks)] = Vec(data[8], data[9], data[10], data[11], data[24], data[25], data[26], data[27]); + values[i_pcolumnv_b(1, 2, block_k, n_blocks)] = Vec(data[40], data[41], data[42], data[43], data[56], data[57], data[58], data[59]); + values[i_pcolumnv_b(0, 3, block_k, n_blocks)] = Vec(data[12], data[13], data[14], data[15], data[28], data[29], data[30], data[31]); + values[i_pcolumnv_b(1, 3, block_k, n_blocks)] = Vec(data[44], data[45], data[46], data[47], data[60], data[61], data[62], data[63]); +#endif //VEC8F_FALLBACK || VEC8D_FALLBACK + #ifdef VEC16F_AGNER values[i_pcolumnv_b(0, 0, block_k, n_blocks)] = gather16f<0 ,1 ,2 ,3 ,16 ,17 ,18 ,19 ,32 ,33 ,34 ,35 ,48 ,49 ,50 ,51>(data); values[i_pcolumnv_b(0, 1, block_k, n_blocks)] = gather16f<4 ,5 ,6 ,7 ,20 ,21 ,22 ,23 ,36 ,37 ,38 ,39 ,52 ,53 ,54 ,55>(data); diff --git a/vlasovsolver/cpu_face_estimates.hpp b/vlasovsolver/cpu_face_estimates.hpp index 1acf96de6..bd2be4f21 100644 --- a/vlasovsolver/cpu_face_estimates.hpp +++ b/vlasovsolver/cpu_face_estimates.hpp @@ -33,136 +33,136 @@ not supported in the solver, so they are now not listed*/ enum face_estimate_order {h4, h5, h6, h8}; -/*! +/*! Compute left face value based on the explicit h8 estimate. Right face value can be obtained as left face value of cell i + 1. - + \param values Array with volume averages. It is assumed a large enough stencil is defined around i. \param i Index of cell in values for which the left face is computed \param fv_l Face value on left face of cell i -*/ -inline void compute_h8_left_face_value(const Vec * const values, uint k, Vec &fv_l){ +*/ +inline void compute_h8_left_face_value(const Vec * const values, uint k, Vec &fv_l){ fv_l = 1.0/840.0 * ( - - 3.0 * values[k - 4] - + 29.0 * values[k - 3] - - 139.0 * values[k - 2] - + 533.0 * values[k - 1] - + 533.0 * values[k] - - 139.0 * values[k + 1] + - 3.0 * values[k - 4] + + 29.0 * values[k - 3] + - 139.0 * values[k - 2] + + 533.0 * values[k - 1] + + 533.0 * values[k] + - 139.0 * values[k + 1] + 29.0 * values[k + 2] - 3.0 * values[k + 3]); } -/*! +/*! Compute left face derivative based on the explicit h7 estimate. Right face derivative can be obtained as left face derivative of cell i + 1. - + \param values Array with volume averages. It is assumed a large enough stencil is defined around i. \param i Index of cell in values for which the left face derivativeis computed \param fd_l Face derivative on left face of cell i -*/ -inline void compute_h7_left_face_derivative(const Vec * const values, uint k, Vec &fd_l){ +*/ +inline void compute_h7_left_face_derivative(const Vec * const values, uint k, Vec &fd_l){ fd_l = 1.0/5040.0 * ( - + 9.0 * values[k - 4] - - 119.0 * values[k - 3] - + 889.0 * values[k - 2] - - 7175.0 * values[k - 1] - + 7175.0 * values[k] - - 889.0 * values[k + 1] + + 9.0 * values[k - 4] + - 119.0 * values[k - 3] + + 889.0 * values[k - 2] + - 7175.0 * values[k - 1] + + 7175.0 * values[k] + - 889.0 * values[k + 1] + 119.0 * values[k + 2] - 9.0 * values[k + 3]); } - -/*! + +/*! Compute left face value based on the explicit h6 estimate. Right face value can be obtained as left face value of cell i + 1. - + \param values Array with volume averages. It is assumed a large enough stencil is defined around i. \param i Index of cell in values for which the left face is computed \param fv_l Face value on left face of cell i -*/ -inline void compute_h6_left_face_value(const Vec * const values, uint k, Vec &fv_l){ +*/ +inline void compute_h6_left_face_value(const Vec * const values, uint k, Vec &fv_l){ /*compute left value*/ - fv_l = 1.0/60.0 * (values[k - 3] - - 8.0 * values[k - 2] - + 37.0 * values[k - 1] - + 37.0 * values[k ] - - 8.0 * values[k + 1] + fv_l = 1.0/60.0 * (values[k - 3] + - 8.0 * values[k - 2] + + 37.0 * values[k - 1] + + 37.0 * values[k ] + - 8.0 * values[k + 1] + values[k + 2]); } -/*! +/*! Compute left face derivative based on the explicit h5 estimate. Right face derivative can be obtained as left face derivative of cell i + 1. - + \param values Array with volume averages. It is assumed a large enough stencil is defined around i. \param i Index of cell in values for which the left face derivativeis computed \param fd_l Face derivative on left face of cell i -*/ -inline void compute_h5_left_face_derivative(const Vec * const values, uint k, Vec &fd_l){ - fd_l = 1.0/180.0 * (245 * (values[k] - values[k - 1]) - - 25 * (values[k + 1] - values[k - 2]) +*/ +inline void compute_h5_left_face_derivative(const Vec * const values, uint k, Vec &fd_l){ + fd_l = 1.0/180.0 * (245 * (values[k] - values[k - 1]) + - 25 * (values[k + 1] - values[k - 2]) + 2 * (values[k + 2] - values[k - 3])); } -/*! +/*! Compute left and right face value based on the explicit h5 estimate. \param values Array with volume averages. It is assumed a large enough stencil is defined around i. \param i Index of cell in values for which the left face is computed \param fv_l Face value on left face of cell i -*/ -inline void compute_h5_face_values(const Vec * const values, uint k, Vec &fv_l, Vec &fv_r){ +*/ +inline void compute_h5_face_values(const Vec * const values, uint k, Vec &fv_l, Vec &fv_r){ /*compute left values*/ - fv_l = 1.0/60.0 * (- 3.0 * values[k - 2] - + 27.0 * values[k - 1] - + 47.0 * values[k ] - - 13.0 * values[k + 1] + fv_l = 1.0/60.0 * (- 3.0 * values[k - 2] + + 27.0 * values[k - 1] + + 47.0 * values[k ] + - 13.0 * values[k + 1] + 2.0 * values[k + 2]); - fv_r = 1.0/60.0 * ( 2.0 * values[k - 2] - - 13.0 * values[k - 1] + fv_r = 1.0/60.0 * ( 2.0 * values[k - 2] + - 13.0 * values[k - 1] + 47.0 * values[k] - + 27.0 * values[k + 1] + + 27.0 * values[k + 1] - 3.0 * values[k + 2]); } -/*! +/*! Compute left face derivative based on the explicit h4 estimate. Right face derivative can be obtained as left face derivative of cell i + 1. - + \param values Array with volume averages. It is assumed a large enough stencil is defined around i. \param i Index of cell in values for which the left face derivativeis computed \param fd_l Face derivative on left face of cell i -*/ -inline void compute_h4_left_face_derivative(const Vec * const values, uint k, Vec &fd_l){ +*/ +inline void compute_h4_left_face_derivative(const Vec * const values, uint k, Vec &fd_l){ fd_l = 1.0/12.0 * (15.0 * (values[k] - values[k - 1]) - (values[k + 1] - values[k - 2])); } -/*! +/*! Compute left face value based on the explicit h4 estimate. Right face value can be obtained as left face value of cell i + 1. - + \param values Array with volume averages. It is assumed a large enough stencil is defined around i. \param i Index of cell in values for which the left face is computed \param fv_l Face value on left face of cell i -*/ -inline void compute_h4_left_face_value(const Vec * const values, uint k, Vec &fv_l){ +*/ +inline void compute_h4_left_face_value(const Vec * const values, uint k, Vec &fv_l){ /*compute left value*/ - fv_l = 1.0/12.0 * ( - 1.0 * values[k - 2] - + 7.0 * values[k - 1] - + 7.0 * values[k] + fv_l = 1.0/12.0 * ( - 1.0 * values[k - 2] + + 7.0 * values[k - 1] + + 7.0 * values[k] - 1.0 * values[k + 1]); } @@ -178,7 +178,7 @@ inline void compute_h4_left_face_value(const Vec * const values, uint k, Vec &fv \param h Array with cell widths. Can be in abritrary units since they always cancel. Maybe 1/refinement ratio? */ inline void compute_h4_left_face_value_nonuniform(const Vec * const h, const Vec * const u, uint k, Vec &fv_l) { - + fv_l = ( 1.0 / ( h[k - 2] + h[k - 1] + h[k] + h[k + 1] ) * ( ( h[k - 2] + h[k - 1] ) * ( h[k] + h[k + 1] ) / ( h[k - 1] + h[k] ) @@ -189,18 +189,18 @@ inline void compute_h4_left_face_value_nonuniform(const Vec * const h, const Vec + h[k - 1] * ( h[k - 2] + h[k - 1] ) / ( ( h[k - 1] + h[k] + h[k + 1] ) * ( h[k] + h[k + 1] ) ) * ( u[k] * ( 2.0 * h[k] + h[k + 1] ) - u[k + 1] * h[k] ) ) ); -} +} -/*! +/*! Compute left face value based on the explicit h4 estimate. Right face value can be obtained as left face value of cell i + 1. - + \param values Array with volume averages. It is assumed a large enough stencil is defined around i. \param i Index of cell in values for which the left face is computed \param fv_l Face value on left face of cell i -*/ -inline void compute_h3_left_face_derivative(const Vec * const values, uint k, Vec &fv_l){ +*/ +inline void compute_h3_left_face_derivative(const Vec * const values, uint k, Vec &fv_l){ /*compute left value*/ fv_l = 1.0/12.0 * (15 * (values[k] - values[k - 1]) - (values[k + 1] - values[k - 2])); } @@ -230,23 +230,27 @@ inline void compute_filtered_face_values_derivatives(const Vec * const values,ui default: case h6: compute_h6_left_face_value(values, k, fv_l); - compute_h6_left_face_value(values, k + 1, fv_r); + compute_h6_left_face_value(values, k + 1, fv_r); compute_h5_left_face_derivative(values, k, fd_l); compute_h5_left_face_derivative(values, k + 1, fd_r); break; case h8: compute_h8_left_face_value(values, k, fv_l); - compute_h8_left_face_value(values, k + 1, fv_r); + compute_h8_left_face_value(values, k + 1, fv_r); compute_h7_left_face_derivative(values, k, fd_l); compute_h7_left_face_derivative(values, k + 1, fd_r); break; } - + Vec slope_abs,slope_sign; - // scale values closer to 1 for more accurate slope limiter calculation - const Realv scale = 1./threshold; - slope_limiter(values[k -1]*scale, values[k]*scale, values[k + 1]*scale, slope_abs, slope_sign); - slope_abs = slope_abs*threshold; + if (threshold>0) { + // scale values closer to 1 for more accurate slope limiter calculation + const Realv scale = 1./threshold; + slope_limiter(values[k -1]*scale, values[k]*scale, values[k + 1]*scale, slope_abs, slope_sign); + slope_abs = slope_abs*threshold; + } else { + slope_limiter(values[k -1], values[k], values[k + 1], slope_abs, slope_sign); + } //check for extrema, flatten if it is Vecb is_extrema = (slope_abs == Vec(0.0)); @@ -259,15 +263,15 @@ inline void compute_filtered_face_values_derivatives(const Vec * const values,ui //Fix left face if needed; boundary value is not bounded or slope is not consistent Vecb filter = (values[k -1] - fv_l) * (fv_l - values[k]) < 0 || slope_sign * fd_l < 0.0; - if(horizontal_or (filter)) { + if(horizontal_or (filter)) { //Go to linear (PLM) estimates if not ok (this is always ok!) fv_l=select(filter, values[k ] - slope_sign * 0.5 * slope_abs, fv_l); fd_l=select(filter, slope_sign * slope_abs, fd_l); } - - //Fix right face if needed; boundary value is not bounded or slope is not consistent + + //Fix right face if needed; boundary value is not bounded or slope is not consistent filter = (values[k + 1] - fv_r) * (fv_r - values[k]) < 0 || slope_sign * fd_r < 0.0; - if(horizontal_or (filter)) { + if(horizontal_or (filter)) { //Go to linear (PLM) estimates if not ok (this is always ok!) fv_r=select(filter, values[k] + slope_sign * 0.5 * slope_abs, fv_r); fd_r=select(filter, slope_sign * slope_abs, fd_r); @@ -294,19 +298,23 @@ inline void compute_filtered_face_values(const Vec * const values,uint k, face_e default: case h6: compute_h6_left_face_value(values, k, fv_l); - compute_h6_left_face_value(values, k + 1, fv_r); + compute_h6_left_face_value(values, k + 1, fv_r); break; case h8: compute_h8_left_face_value(values, k, fv_l); - compute_h8_left_face_value(values, k + 1, fv_r); + compute_h8_left_face_value(values, k + 1, fv_r); break; } Vec slope_abs,slope_sign; - // scale values closer to 1 for more accurate slope limiter calculation - const Realv scale = 1./threshold; - slope_limiter(values[k -1]*scale, values[k]*scale, values[k + 1]*scale, slope_abs, slope_sign); - slope_abs = slope_abs*threshold; - + if (threshold>0) { + // scale values closer to 1 for more accurate slope limiter calculation + const Realv scale = 1./threshold; + slope_limiter(values[k -1]*scale, values[k]*scale, values[k + 1]*scale, slope_abs, slope_sign); + slope_abs = slope_abs*threshold; + } else { + slope_limiter(values[k -1], values[k], values[k + 1], slope_abs, slope_sign); + } + //check for extrema, flatten if it is Vecb is_extrema = (slope_abs == Vec(0.0)); if(horizontal_or(is_extrema)) { @@ -316,14 +324,14 @@ inline void compute_filtered_face_values(const Vec * const values,uint k, face_e //Fix left face if needed; boundary value is not bounded Vecb filter = (values[k -1] - fv_l) * (fv_l - values[k]) < 0 ; - if(horizontal_or (filter)) { + if(horizontal_or (filter)) { //Go to linear (PLM) estimates if not ok (this is always ok!) fv_l=select(filter, values[k ] - slope_sign * 0.5 * slope_abs, fv_l); } - //Fix face if needed; boundary value is not bounded + //Fix face if needed; boundary value is not bounded filter = (values[k + 1] - fv_r) * (fv_r - values[k]) < 0; - if(horizontal_or (filter)) { + if(horizontal_or (filter)) { //Go to linear (PLM) estimates if not ok (this is always ok!) fv_r=select(filter, values[k] + slope_sign * 0.5 * slope_abs, fv_r); } @@ -332,7 +340,7 @@ inline void compute_filtered_face_values(const Vec * const values,uint k, face_e inline void compute_filtered_face_values_nonuniform(const Vec * const dv, const Vec * const values,uint k, face_estimate_order order, Vec &fv_l, Vec &fv_r, const Realv threshold){ switch(order){ - case h4: + case h4: compute_h4_left_face_value_nonuniform(dv, values, k, fv_l); compute_h4_left_face_value_nonuniform(dv, values, k + 1, fv_r); break; @@ -341,22 +349,26 @@ inline void compute_filtered_face_values_nonuniform(const Vec * const dv, const // break; // case h6: // compute_h6_left_face_value(dv, values, k, fv_l); - // compute_h6_left_face_value(dv, values, k + 1, fv_r); + // compute_h6_left_face_value(dv, values, k + 1, fv_r); // break; // case h8: // compute_h8_left_face_value(dv, values, k, fv_l); - // compute_h8_left_face_value(dv, values, k + 1, fv_r); + // compute_h8_left_face_value(dv, values, k + 1, fv_r); // break; default: std::cout << "Order " << order << " has not been implemented (yet)\n"; break; } Vec slope_abs,slope_sign; - // scale values closer to 1 for more accurate slope limiter calculation - const Realv scale = 1./threshold; - slope_limiter(values[k -1]*scale, values[k]*scale, values[k + 1]*scale, slope_abs, slope_sign); - slope_abs = slope_abs*threshold; - + if (threshold>0) { + // scale values closer to 1 for more accurate slope limiter calculation + const Realv scale = 1./threshold; + slope_limiter(values[k -1]*scale, values[k]*scale, values[k + 1]*scale, slope_abs, slope_sign); + slope_abs = slope_abs*threshold; + } else { + slope_limiter(values[k -1], values[k], values[k + 1], slope_abs, slope_sign); + } + //check for extrema, flatten if it is Vecb is_extrema = (slope_abs == Vec(0.0)); if(horizontal_or(is_extrema)) { @@ -366,14 +378,14 @@ inline void compute_filtered_face_values_nonuniform(const Vec * const dv, const //Fix left face if needed; boundary value is not bounded Vecb filter = (values[k -1] - fv_l) * (fv_l - values[k]) < 0 ; - if(horizontal_or (filter)) { + if(horizontal_or (filter)) { //Go to linear (PLM) estimates if not ok (this is always ok!) fv_l=select(filter, values[k ] - slope_sign * 0.5 * slope_abs, fv_l); } - //Fix face if needed; boundary value is not bounded + //Fix face if needed; boundary value is not bounded filter = (values[k + 1] - fv_r) * (fv_r - values[k]) < 0; - if(horizontal_or (filter)) { + if(horizontal_or (filter)) { //Go to linear (PLM) estimates if not ok (this is always ok!) fv_r=select(filter, values[k] + slope_sign * 0.5 * slope_abs, fv_r); } @@ -395,14 +407,14 @@ inline Vec get_D2aLim(const Vec * h, const Vec * values, uint k, const Vec C, Ve } return d2aLim; - + } inline pair constrain_face_values(const Vec * h,const Vec * values,uint k,Vec & fv_l, Vec & fv_r) { const Vec C = 1.25; Vec invh2 = 1.0 / (h[k] * h[k]); - + // Colella & Sekora, eq 19 Vec p_face = 0.5 * (values[k] + values[k + 1]) - h[k] * h[k] / 3.0 * get_D2aLim(h,values,k ,C,fv_r); @@ -410,7 +422,7 @@ inline pair constrain_face_values(const Vec * h,const Vec * values,uint - h[k-1] * h[k-1] / 3.0 * get_D2aLim(h,values,k-1,C,fv_l); // Colella & Sekora, eq 21 - Vec d2a = -2.0 * invh2 * 6.0 * (values[k] - 3.0 * (m_face + p_face)); // a6,j from eq. 7 + Vec d2a = -2.0 * invh2 * 6.0 * (values[k] - 3.0 * (m_face + p_face)); // a6,j from eq. 7 Vec d2aC = invh2 * (values[k - 1] - 2.0 * values[k ] + values[k + 1]); // Note: Corrected the index of 2nd term in d2aL to k - 1. // In the paper it is k but that is almost certainly an error. @@ -421,7 +433,7 @@ inline pair constrain_face_values(const Vec * h,const Vec * values,uint // Colella & Sekora, eq 22 if ( (horizontal_or(d2a * d2aL >= 0)) && (horizontal_or(d2a * d2aR >= 0)) && (horizontal_or(d2a * d2aC >= 0)) && horizontal_and(d2a != 0)) { - + d2aLim = d2a / abs(d2a) * min(C * abs(d2aL), min(C * abs(d2aR), min(C * abs(d2aC), abs(d2a)))); } else { d2aLim = 0.0; @@ -431,95 +443,98 @@ inline pair constrain_face_values(const Vec * h,const Vec * values,uint d2a = 1.0; } } - + // Colella & Sekora, eq 23 Vec p_face_interpolant = values[k] + (p_face - values[k]) * d2aLim / d2a; Vec m_face_interpolant = values[k] + (m_face - values[k]) * d2aLim / d2a; - + pair faceInterpolants; faceInterpolants = make_pair(m_face_interpolant,p_face_interpolant); - + // if(horizontal_and(values[k] > 1.0)) // std::cout << "k, a-, a+, : " << k << ", " << p_face_interpolant[0] << ", " << m_face_interpolant[0] << ", " << values[k][0] << endl; - + return faceInterpolants; } inline void compute_filtered_face_values_nonuniform_conserving(const Vec * const dv, const Vec * const values,uint k, face_estimate_order order, Vec &fv_l, Vec &fv_r, const Realv threshold){ - switch(order){ - case h4: - compute_h4_left_face_value_nonuniform(dv, values, k, fv_l); - compute_h4_left_face_value_nonuniform(dv, values, k + 1, fv_r); - break; - // case h5: - // compute_h5_face_values(dv, values, k, fv_l, fv_r); - // break; - // case h6: - // compute_h6_left_face_value(dv, values, k, fv_l); - // compute_h6_left_face_value(dv, values, k + 1, fv_r); - // break; - // case h8: - // compute_h8_left_face_value(dv, values, k, fv_l); - // compute_h8_left_face_value(dv, values, k + 1, fv_r); - // break; - default: - std::cout << "Order " << order << " has not been implemented (yet)\n"; - break; - } + switch(order){ + case h4: + compute_h4_left_face_value_nonuniform(dv, values, k, fv_l); + compute_h4_left_face_value_nonuniform(dv, values, k + 1, fv_r); + break; + // case h5: + // compute_h5_face_values(dv, values, k, fv_l, fv_r); + // break; + // case h6: + // compute_h6_left_face_value(dv, values, k, fv_l); + // compute_h6_left_face_value(dv, values, k + 1, fv_r); + // break; + // case h8: + // compute_h8_left_face_value(dv, values, k, fv_l); + // compute_h8_left_face_value(dv, values, k + 1, fv_r); + // break; + default: + std::cout << "Order " << order << " has not been implemented (yet)\n"; + break; + } - Vec slope_abs,slope_sign; - // scale values closer to 1 for more accurate slope limiter calculation - const Realv scale = 1./threshold; - slope_limiter(values[k -1]*scale, values[k]*scale, values[k + 1]*scale, slope_abs, slope_sign); - slope_abs = slope_abs*threshold; - - //check for extrema - Vecb is_extrema = (slope_abs == Vec(0.0)); - Vecb filter_l = (values[k - 1] - fv_l) * (fv_l - values[k]) < 0 ; - Vecb filter_r = (values[k + 1] - fv_r) * (fv_r - values[k]) < 0; - // if(horizontal_or(is_extrema) || horizontal_or(filter_l) || horizontal_or(filter_r)) { - // Colella & Sekora, eq. 20 - if(horizontal_or((fv_r - values[k]) * (values[k] - fv_l) <= Vec(0.0)) - && horizontal_or((values[k - 1] - values[k]) * (values[k] - values[k + 1]) <= Vec(0.0))) { - auto faces = constrain_face_values(dv, values, k, fv_l, fv_r); - - fv_l = faces.first; - fv_r = faces.second; - - // fv_r = select(is_extrema, values[k], fv_r); - // fv_l = select(is_extrema, values[k], fv_l); - } else { + Vec slope_abs,slope_sign; + if (threshold>0) { + // scale values closer to 1 for more accurate slope limiter calculation + const Realv scale = 1./threshold; + slope_limiter(values[k -1]*scale, values[k]*scale, values[k + 1]*scale, slope_abs, slope_sign); + slope_abs = slope_abs*threshold; + } else { + slope_limiter(values[k -1], values[k], values[k + 1], slope_abs, slope_sign); + } + + //check for extrema + Vecb is_extrema = (slope_abs == Vec(0.0)); + Vecb filter_l = (values[k - 1] - fv_l) * (fv_l - values[k]) < 0 ; + Vecb filter_r = (values[k + 1] - fv_r) * (fv_r - values[k]) < 0; + // if(horizontal_or(is_extrema) || horizontal_or(filter_l) || horizontal_or(filter_r)) { + // Colella & Sekora, eq. 20 + if(horizontal_or((fv_r - values[k]) * (values[k] - fv_l) <= Vec(0.0)) + && horizontal_or((values[k - 1] - values[k]) * (values[k] - values[k + 1]) <= Vec(0.0))) { + auto faces = constrain_face_values(dv, values, k, fv_l, fv_r); + + fv_l = faces.first; + fv_r = faces.second; + + // fv_r = select(is_extrema, values[k], fv_r); + // fv_l = select(is_extrema, values[k], fv_l); + } else { + + //Fix left face if needed; boundary value is not bounded + Vecb filter = (values[k -1] - fv_l) * (fv_l - values[k]) < 0 ; + if(horizontal_or (filter)) { + //Go to linear (PLM) estimates if not ok (this is always ok!) + fv_l=select(filter, values[k ] - slope_sign * 0.5 * slope_abs, fv_l); + } + + //Fix face if needed; boundary value is not bounded + filter = (values[k + 1] - fv_r) * (fv_r - values[k]) < 0; + if(horizontal_or (filter)) { + //Go to linear (PLM) estimates if not ok (this is always ok!) + fv_r=select(filter, values[k] + slope_sign * 0.5 * slope_abs, fv_r); + } + } + + // //Fix left face if needed; boundary value is not bounded + // Vecb filter = (values[k -1] - fv_l) * (fv_l - values[k]) < 0 ; + // if(horizontal_or (filter)) { + // //Go to linear (PLM) estimates if not ok (this is always ok!) + // fv_l=select(filter, values[k ] - slope_sign * 0.5 * slope_abs, fv_l); + // } + + // //Fix face if needed; boundary value is not bounded + // filter = (values[k + 1] - fv_r) * (fv_r - values[k]) < 0; + // if(horizontal_or (filter)) { + // //Go to linear (PLM) estimates if not ok (this is always ok!) + // fv_r=select(filter, values[k] + slope_sign * 0.5 * slope_abs, fv_r); + // } - //Fix left face if needed; boundary value is not bounded - Vecb filter = (values[k -1] - fv_l) * (fv_l - values[k]) < 0 ; - if(horizontal_or (filter)) { - //Go to linear (PLM) estimates if not ok (this is always ok!) - fv_l=select(filter, values[k ] - slope_sign * 0.5 * slope_abs, fv_l); - } - - //Fix face if needed; boundary value is not bounded - filter = (values[k + 1] - fv_r) * (fv_r - values[k]) < 0; - if(horizontal_or (filter)) { - //Go to linear (PLM) estimates if not ok (this is always ok!) - fv_r=select(filter, values[k] + slope_sign * 0.5 * slope_abs, fv_r); - } - } - - // //Fix left face if needed; boundary value is not bounded - // Vecb filter = (values[k -1] - fv_l) * (fv_l - values[k]) < 0 ; - // if(horizontal_or (filter)) { - // //Go to linear (PLM) estimates if not ok (this is always ok!) - // fv_l=select(filter, values[k ] - slope_sign * 0.5 * slope_abs, fv_l); - // } - - // //Fix face if needed; boundary value is not bounded - // filter = (values[k + 1] - fv_r) * (fv_r - values[k]) < 0; - // if(horizontal_or (filter)) { - // //Go to linear (PLM) estimates if not ok (this is always ok!) - // fv_r=select(filter, values[k] + slope_sign * 0.5 * slope_abs, fv_r); - // } - - } diff --git a/vlasovsolver/cpu_trans_map_amr.cpp b/vlasovsolver/cpu_trans_map_amr.cpp index 935363489..12491a02a 100644 --- a/vlasovsolver/cpu_trans_map_amr.cpp +++ b/vlasovsolver/cpu_trans_map_amr.cpp @@ -7,6 +7,8 @@ #include "cpu_trans_map_amr.hpp" #include "cpu_trans_map.hpp" +// use DCCRG version Nov 8th 2018 01482cfba8 + using namespace std; using namespace spatial_cell; @@ -50,7 +52,7 @@ int getNeighborhood(const uint dimension, const uint stencil) { } } - if (stencil == 2) { + if (stencil > 1) { switch (dimension) { case 0: neighborhood = VLASOV_SOLVER_X_NEIGHBORHOOD_ID; @@ -69,7 +71,6 @@ int getNeighborhood(const uint dimension, const uint stencil) { } - /* Get pointers to spatial cells that are considered source cells for a pencil. * Source cells are cells that the pencil reads data from to compute polynomial * fits that are used for propagation in the vlasov solver. All cells included @@ -92,48 +93,93 @@ void computeSpatialSourceCellsForPencil(const dccrg::Dccrg ids = pencils.getIds(iPencil); - - int neighborhood = getNeighborhood(dimension,2); + + // These neighborhoods now include the AMR addition beyond the regular vlasov stencil + int neighborhood = getNeighborhood(dimension,VLASOV_STENCIL_WIDTH); // Get pointers for each cell id of the pencil for (int i = 0; i < L; ++i) { sourceCells[i + VLASOV_STENCIL_WIDTH] = mpiGrid[ids[i]]; } - + // Insert pointers for neighbors of ids.front() and ids.back() const auto* frontNbrPairs = mpiGrid.get_neighbors_of(ids.front(), neighborhood); const auto* backNbrPairs = mpiGrid.get_neighbors_of(ids.back(), neighborhood); int maxRefLvl = mpiGrid.get_maximum_refinement_level(); - + // Create list of unique distances in the negative direction from the first cell in pencil std::set< int > distances; for (const auto nbrPair : *frontNbrPairs) { if(nbrPair.second[dimension] < 0) { - distances.insert(nbrPair.second[dimension]); + // gather positive distance values + distances.insert(-nbrPair.second[dimension]); } } - int iSrc = VLASOV_STENCIL_WIDTH - 1; + // Iterate through distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. - // Distances are negative here so largest distance has smallest value - auto irend = distances.rbegin(); - std::advance(irend, std::min((int)distances.size(), VLASOV_STENCIL_WIDTH)); - for (auto it = distances.rbegin(); it != irend; ++it) { + for (auto it = distances.begin(); it != distances.end(); ++it) { + if (iSrc < 0) break; // found enough elements + // Collect all neighbors at distance *it to a vector std::vector< CellID > neighbors; for (const auto nbrPair : *frontNbrPairs) { - int distanceInRefinedCells = nbrPair.second[dimension]; + int distanceInRefinedCells = -nbrPair.second[dimension]; if(distanceInRefinedCells == *it) neighbors.push_back(nbrPair.first); } + // Get rid of duplicate neighbor cells at single distance + neighbors.erase(unique(neighbors.begin(), neighbors.end()), neighbors.end()); int refLvl = mpiGrid.get_refinement_level(ids.front()); - if (neighbors.size() == 1) { + if (sourceCells[iSrc+1] == mpiGrid[neighbors.at(0)]) continue; // already found this cell for different distance sourceCells[iSrc--] = mpiGrid[neighbors.at(0)]; } else if ( pencils.path[iPencil][refLvl] < neighbors.size() ) { + if (sourceCells[iSrc+1] == mpiGrid[neighbors.at(pencils.path[iPencil][refLvl])]) continue; // already found this cell for different distance (should not happen) sourceCells[iSrc--] = mpiGrid[neighbors.at(pencils.path[iPencil][refLvl])]; + // Code for alternate approach to verify that multiple neighbors are in correct ordering (z-y-x) + // int ix=0,iy=0; + // switch(dimension) { + // case 0: + // ix = 1; + // iy = 2; + // break; + // case 1: + // ix = 0; + // iy = 2; + // break; + // case 2: + // ix = 0; + // iy = 1; + // break; + // } + // bool accept = false; + // std::array parentCoords = mpiGrid.get_center(ids.front()); + // for (CellID n : neighbors) { + // std::array myCoords = mpiGrid.get_center(n); + // switch (pencils.path[iPencil][refLvl]) { + // case 0: + // if (myCoords[ix] < parentCoords[ix] && myCoords[iy] < parentCoords[iy]) accept=true; + // break; + // case 1: + // if (myCoords[ix] > parentCoords[ix] && myCoords[iy] < parentCoords[iy]) accept=true; + // break; + // case 2: + // if (myCoords[ix] < parentCoords[ix] && myCoords[iy] > parentCoords[iy]) accept=true; + // break; + // case 3: + // if (myCoords[ix] > parentCoords[ix] && myCoords[iy] > parentCoords[iy]) accept=true; + // break; + // } + // if (accept) { + // sourceCells[iSrc--] = mpiGrid[n]; + // break; + // } + // } + } else { + std::cerr<<"error too few neighbors for path! "<= L+2*VLASOV_STENCIL_WIDTH) break; // Found enough cells + // Collect all neighbors at distance *it to a vector std::vector< CellID > neighbors; for (const auto nbrPair : *backNbrPairs) { int distanceInRefinedCells = nbrPair.second[dimension]; if(distanceInRefinedCells == *it) neighbors.push_back(nbrPair.first); } + // Get rid of duplicate neighbor cells at single distance + neighbors.erase(unique(neighbors.begin(), neighbors.end()), neighbors.end()); int refLvl = mpiGrid.get_refinement_level(ids.back()); - if (neighbors.size() == 1) { + if (sourceCells[iSrc-1] == mpiGrid[neighbors.at(0)]) continue; // already found this cell for different distance sourceCells[iSrc++] = mpiGrid[neighbors.at(0)]; } else if ( pencils.path[iPencil][refLvl] < neighbors.size() ) { + if (sourceCells[iSrc-1] == mpiGrid[neighbors.at(pencils.path[iPencil][refLvl])]) continue; // already found this cell for different distance (should not happen) sourceCells[iSrc++] = mpiGrid[neighbors.at(pencils.path[iPencil][refLvl])]; + } else { + std::cerr<<"error too few neighbors for path!"<= 0 ;i--){ - if(sourceCells[i] == NULL) + for(int i = VLASOV_STENCIL_WIDTH - 1; i >= 0 ;--i){ + if(sourceCells[i] == NULL || sourceCells[i]->sysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE) sourceCells[i] = lastGoodCell; else lastGoodCell = sourceCells[i]; } /*loop to positive side and replace all invalid cells with the closest good cell*/ lastGoodCell = mpiGrid[ids.back()]; - for(int i = L + VLASOV_STENCIL_WIDTH; i < L + 2*VLASOV_STENCIL_WIDTH; i++){ - if(sourceCells[i] == NULL) + for(int i = VLASOV_STENCIL_WIDTH + L; i < (L + 2*VLASOV_STENCIL_WIDTH); ++i){ + if(sourceCells[i] == NULL || sourceCells[i]->sysBoundaryFlag == sysboundarytype::DO_NOT_COMPUTE) sourceCells[i] = lastGoodCell; else lastGoodCell = sourceCells[i]; } } + /* Get pointers to spatial cells that are considered target cells for all pencils. * Target cells are cells that the pencil writes data into after translation by * the vlasov solver. All cells included in the pencil + 1 cells on both ends * are source cells. Boundary cells are not included. + * Now uses get_face_neighbors_of(). * * @param [in] mpiGrid DCCRG grid object * @param [in] pencils pencil data struct * @param [in] dimension spatial dimension - * @param [out] sourceCells pointer to an array of pointers to SpatialCell objects for the target cells + * @param [out] targetCells pointer to an array of pointers to SpatialCell objects for the target cells * */ -void computeSpatialTargetCellsForPencils(const dccrg::Dccrg& mpiGrid, +void computeSpatialTargetCellsForPencilsWithFaces(const dccrg::Dccrg& mpiGrid, setOfPencils& pencils, const uint dimension, SpatialCell **targetCells){ - int neighborhood = getNeighborhood(dimension,1); - uint GID = 0; // Loop over pencils for(uint iPencil = 0; iPencil < pencils.N; iPencil++){ - int L = pencils.lengthOfPencils[iPencil]; vector ids = pencils.getIds(iPencil); - + // Get pointers for each cell id of the pencil for (int i = 0; i < L; ++i) { targetCells[GID + i + 1] = mpiGrid[ids[i]]; } - // Insert pointers for neighbors of ids.front() and ids.back() - auto frontNbrPairs = mpiGrid.get_neighbors_of(ids.front(), neighborhood); - auto backNbrPairs = mpiGrid.get_neighbors_of(ids.back(), neighborhood); - + int refLvl; vector frontNeighborIds; - for( const auto nbrPair: *frontNbrPairs ) { - if (nbrPair.second.at(dimension) == -1) { - frontNeighborIds.push_back(nbrPair.first); - } - } - - if (frontNeighborIds.size() == 0) { - abort(); - } - vector backNeighborIds; - for( const auto nbrPair: *backNbrPairs ) { - if (nbrPair.second.at(dimension) == 1) { - backNeighborIds.push_back(nbrPair.first); + const auto frontNeighbors = mpiGrid.get_face_neighbors_of(ids.front()); + if (frontNeighbors.size() > 0) { + for (const auto nbr: frontNeighbors) { + if(nbr.second == (-((int)dimension + 1))) { + frontNeighborIds.push_back(nbr.first); + } } + refLvl = mpiGrid.get_refinement_level(ids.front()); + + if (frontNeighborIds.size() == 0) { + std::cerr<<"abort frontNeighborIds.size() == 0 at "< 0) { + for (const auto nbr: backNeighbors) { + if(nbr.second == ((int)dimension + 1)) { + backNeighborIds.push_back(nbr.first); + } + } + refLvl = mpiGrid.get_refinement_level(ids.back()); + if (backNeighborIds.size() == 0) { + std::cerr<<"abort backNeighborIds.size() == 0 at "< &grid, CellID id, int dimension = 0, uint path = 0) { - int neighborhood = getNeighborhood(dimension,1); - - const auto* nbrPairs = grid.get_neighbors_of(id, neighborhood); + //int neighborhood = getNeighborhood(dimension,1); + //const auto* nbrPairs = grid.get_neighbors_of(id, neighborhood); vector < CellID > myNeighbors; CellID neighbor = INVALID_CELLID; // Iterate through neighbor ids in the positive direction of the chosen dimension, // select the neighbor indicated by path, if it is local to this process. - for (const auto nbrPair : *nbrPairs) { - if (nbrPair.second[dimension] == 1) { - myNeighbors.push_back(nbrPair.first); + const auto faceNbrs = grid.get_face_neighbors_of(id); + for (const auto nbr : faceNbrs) { + if (nbr.second == ((int)dimension + 1)) { + myNeighbors.push_back(nbr.first); } } - + if( myNeighbors.size() == 0 ) { return neighbor; } @@ -309,7 +367,7 @@ CellID selectNeighbor(const dccrg::Dccrg } if (grid.is_local(myNeighbors[neighborIndex])) { - neighbor = myNeighbors[neighborIndex]; + neighbor = myNeighbors[neighborIndex]; } return neighbor; @@ -343,6 +401,7 @@ setOfPencils buildPencilsWithNeighbors( const dccrg::Dccrg coordinates = grid.get_center(seedId); int startingPathSize = path.size(); auto it = path.end(); if( startingRefLvl > startingPathSize ) { @@ -359,23 +419,40 @@ setOfPencils buildPencilsWithNeighbors( const dccrg::Dccrg parentCoords.at(ix) && myCoords.at(iy) < parentCoords.at(iy)) { + } else if (myCoords[ix] > parentCoords[ix] && myCoords[iy] < parentCoords[iy]) { step = 1; - } else if (myCoords.at(ix) < parentCoords.at(ix) && myCoords.at(iy) > parentCoords.at(iy)) { + } else if (myCoords[ix] < parentCoords[ix] && myCoords[iy] > parentCoords[iy]) { step = 2; - } else if (myCoords.at(ix) > parentCoords.at(ix) && myCoords.at(iy) > parentCoords.at(iy)) { + } else if (myCoords[ix] > parentCoords[ix] && myCoords[iy] > parentCoords[iy]) { step = 3; } @@ -421,7 +498,8 @@ setOfPencils buildPencilsWithNeighbors( const dccrg::Dccrg= nTargetNeighborsPerPencil is required (default 2 >= 1) - - for (uint i = 0; i < lengthOfPencil + 2 * nTargetNeighborsPerPencil; i++) { - - for (uint k = 0; k < WID; ++k) { - - for (uint planeVector = 0; planeVector < VEC_PER_PLANE; planeVector++) { - int im1 = i - 1; // doing this to shut up compiler warnings - values[i_trans_ps_blockv_pencil(planeVector, k, im1, lengthOfPencil)] = - targetValues[i_trans_pt_blockv(planeVector, k, im1)]; - - } - } - } } /* Determine which cells in the local DCCRG mesh should be starting points for pencils. @@ -647,34 +722,33 @@ void getSeedIds(const dccrg::Dccrg& mpiGr auto myIndices = mpiGrid.mapping.get_indices(celli); -#warning This forces single-cell pencils! - // FIXME TODO Tuomas look at this! BUG - bool addToSeedIds = true; - // Returns all neighbors as (id, direction-dimension) pair pointers. - for ( const auto nbrPair : *(mpiGrid.get_neighbors_of(celli, neighborhood)) ) { - - if ( nbrPair.second[dimension] == -1 ) { + bool addToSeedIds = P::transShortPencils; + if (addToSeedIds) { + seedIds.push_back(celli); + continue; + } - // Check that the neighbor is not across a periodic boundary by calculating - // the distance in indices between this cell and its neighbor. - auto nbrIndices = mpiGrid.mapping.get_indices(nbrPair.first); - - // If a neighbor is non-local, across a periodic boundary, or in non-periodic boundary layer 1 - // then we use this cell as a seed for pencils - if ( abs ( (int64_t)(myIndices[dimension] - nbrIndices[dimension]) ) > - pow(2,mpiGrid.get_maximum_refinement_level()) || - !mpiGrid.is_local(nbrPair.first) || - !do_translate_cell(mpiGrid[nbrPair.first]) ) { - addToSeedIds = true; - } - } + // Returns all neighbors as (id, direction-dimension) pair pointers. + for ( const auto nbrPair : mpiGrid.get_face_neighbors_of(celli) ) { + if ( nbrPair.second == -((int)dimension + 1) ) { + // Check that the neighbor is not across a periodic boundary by calculating + // the distance in indices between this cell and its neighbor. + auto nbrIndices = mpiGrid.mapping.get_indices(nbrPair.first); + + // If a neighbor is non-local, across a periodic boundary, or in non-periodic boundary layer 1 + // then we use this cell as a seed for pencils + if ( abs ( (int64_t)(myIndices[dimension] - nbrIndices[dimension]) ) > + pow(2,mpiGrid.get_maximum_refinement_level()) || + !mpiGrid.is_local(nbrPair.first) || + !do_translate_cell(mpiGrid[nbrPair.first]) ) { + addToSeedIds = true; + } + } } if ( addToSeedIds ) { seedIds.push_back(celli); } - - //cout << endl; } if(debug) { @@ -787,9 +861,7 @@ bool copy_trans_block_data_amr( } /* Check whether the ghost cells around the pencil contain higher refinement than the pencil does. - * If they do, the pencil must be split to match the finest refined ghost cell. This function checks - * One neighbor pair, but takes as an argument the offset from the pencil. Call multiple times for - * Multiple ghost cells. + * If they do, the pencil must be split to match the finest refined ghost cell. * * @param mpiGrid DCCRG grid object * @param pencils Pencil data struct @@ -808,7 +880,9 @@ void check_ghost_cells(const dccrg::Dccrg } std::vector idsToSplit; - + +// Thread this loop here +#pragma omp parallel for for (uint pencili = 0; pencili < pencils.N; ++pencili) { if(pencils.periodic[pencili]) continue; @@ -818,20 +892,70 @@ void check_ghost_cells(const dccrg::Dccrg // It is possible that the pencil has already been refined by the pencil building algorithm // and is on a higher refinement level than the refinement level of any of the cells it contains // due to e.g. process boundaries. - int maxPencilRefLvl = pencils.path.at(pencili).size(); + int maxPencilRefLvl = pencils.path[pencili].size(); int maxNbrRefLvl = 0; - + const auto* frontNeighbors = mpiGrid.get_neighbors_of(ids.front(),neighborhoodId); const auto* backNeighbors = mpiGrid.get_neighbors_of(ids.back() ,neighborhoodId); - for (const auto nbrPair: *frontNeighbors) { - maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); + + // Create list of unique distances in the negative direction from the first cell in pencil + std::set< int > distances; + for (const auto nbrPair : *frontNeighbors) { + if(nbrPair.second[dimension] < 0) { + // gather positive values + distances.insert(-nbrPair.second[dimension]); + } } - - for (const auto nbrPair: *backNeighbors) { - maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); + int foundcells = 0; + CellID lastcell = INVALID_CELLID; + // Iterate through distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. + for (auto it = distances.begin(); it != distances.end(); ++it) { + for (const auto nbrPair : *frontNeighbors) { + if (nbrPair.first==lastcell) continue; + int distanceInRefinedCells = -nbrPair.second[dimension]; + if(distanceInRefinedCells == *it) { + maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); + lastcell = nbrPair.first; + foundcells++; + continue; + } + } + if (foundcells >= VLASOV_STENCIL_WIDTH) break; // checked enough distances + } + + // Create list of unique distances in the positive direction from the last cell in pencil + distances.clear(); + for (const auto nbrPair : *backNeighbors) { + if(nbrPair.second[dimension] > 0) { + distances.insert(nbrPair.second[dimension]); + } + } + foundcells = 0; + lastcell = INVALID_CELLID; + for (auto it = distances.begin(); it != distances.end(); ++it) { + for (const auto nbrPair : *backNeighbors) { + if (nbrPair.first==lastcell) continue; + int distanceInRefinedCells = nbrPair.second[dimension]; + if(distanceInRefinedCells == *it) { + maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); + lastcell = nbrPair.first; + foundcells++; + continue; + } + } + if (foundcells >= VLASOV_STENCIL_WIDTH) break; // checked enough distances } + // Old version which can check needlessly far + // for (const auto nbrPair: *frontNeighbors) { + // maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); + // } + // for (const auto nbrPair: *backNeighbors) { + // maxNbrRefLvl = max(maxNbrRefLvl,mpiGrid.get_refinement_level(nbrPair.first)); + // } + + if (maxNbrRefLvl > maxPencilRefLvl) { if(debug) { std::cout << "I am rank " << myRank << ". "; @@ -841,10 +965,14 @@ void check_ghost_cells(const dccrg::Dccrg } // Let's avoid modifying pencils while we are looping over it. Write down the indices of pencils // that need to be split and split them later. - idsToSplit.push_back(pencili); +#pragma omp critical + { + idsToSplit.push_back(pencili); + } } } +// No threading here, probably more efficient to thread inside the splitting for (auto pencili: idsToSplit) { Realv dx = 0.0; @@ -857,8 +985,8 @@ void check_ghost_cells(const dccrg::Dccrg dy = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DZ]; break; case 1: - dx = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DZ]; - dy = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DX]; + dx = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DX]; + dy = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DZ]; break; case 2: dx = mpiGrid[ids[0]]->SpatialCell::parameters[CellParams::DX]; @@ -866,6 +994,7 @@ void check_ghost_cells(const dccrg::Dccrg break; } +// WARNING threading inside this function pencils.split(pencili,dx,dy); } @@ -901,7 +1030,7 @@ bool checkPencils( } - for (int ipencil = 0; ipencil < pencils.N; ++ipencil) { + for (uint ipencil = 0; ipencil < pencils.N; ++ipencil) { cint nPencilsThroughThisCell = pow(pow(2,pencils.path[ipencil].size()),2); auto ids = pencils.getIds(ipencil); @@ -1057,7 +1186,7 @@ bool trans_map_1d_amr(const dccrg::Dccrg& // Output vectors for ready pencils setOfPencils pencils; -#pragma omp parallel + #pragma omp parallel { // Empty vectors for internal use of buildPencilsWithNeighbors. Could be default values but // default vectors are complicated. Should overload buildPencilsWithNeighbors like suggested here @@ -1069,7 +1198,7 @@ bool trans_map_1d_amr(const dccrg::Dccrg& // iterators used in the accumulation std::vector::iterator ibeg, iend; -#pragma omp for schedule(guided) + #pragma omp for schedule(guided) for (uint i=0; i& } // accumulate thread results in global set of pencils -#pragma omp critical + #pragma omp critical { - for (uint i=0; i pencilIds(ibeg, iend); @@ -1089,7 +1218,7 @@ bool trans_map_1d_amr(const dccrg::Dccrg& } // init cellid_transpose (moved here to take advantage of the omp parallel region) -#pragma omp for collapse(3) + #pragma omp for collapse(3) for (uint k=0; k& } } + phiprof::start("check_ghost_cells"); // Check refinement of two ghost cells on each end of each pencil check_ghost_cells(mpiGrid,pencils,dimension); + phiprof::stop("check_ghost_cells"); + // **************************************************************************** if(printPencils) printPencilsFunc(pencils,dimension,myRank); - if(!checkPencils(mpiGrid, localPropagatedCells, pencils)) { - abort(); - } + // Warning: checkPencils fails to understand situations where pencils reach across 3 levels of refinement. + // if(!checkPencils(mpiGrid, localPropagatedCells, pencils)) { + // std::cerr<<"abort checkpencils"<& // be the most nice way to do this and in any case we could do it along // dimension for data locality reasons => copy acc map column code, TODO: FIXME // TODO: Do this separately for each pencil? + std::vector unionOfBlocks; std::unordered_set unionOfBlocksSet; - - for(auto cell : allCellsPointer) { - vmesh::VelocityMesh& cvmesh = cell->get_velocity_mesh(popID); - for (vmesh::LocalID block_i=0; block_i< cvmesh.size(); ++block_i) { - unionOfBlocksSet.insert(cvmesh.getGlobalID(block_i)); +// unionOfBlocks.reserve(unionOfBlocksSet.size()); +#pragma omp parallel + { + std::unordered_set thread_unionOfBlocksSet; + +#pragma omp for + for(unsigned int i=0; i& cvmesh = (*cell)->get_velocity_mesh(popID); + for (vmesh::LocalID block_i=0; block_i< cvmesh.size(); ++block_i) { + thread_unionOfBlocksSet.insert(cvmesh.getGlobalID(block_i)); + } } - } + +#pragma omp critical + { + unionOfBlocksSet.insert(thread_unionOfBlocksSet.begin(), thread_unionOfBlocksSet.end()); + } // pragma omp critical + } // pragma omp parallel + + unionOfBlocks.insert(unionOfBlocks.end(), unionOfBlocksSet.begin(), unionOfBlocksSet.end()); - std::vector unionOfBlocks; - unionOfBlocks.reserve(unionOfBlocksSet.size()); - for(const auto blockGID: unionOfBlocksSet) { - unionOfBlocks.push_back(blockGID); - } phiprof::stop("buildBlockList"); // **************************************************************************** @@ -1156,7 +1300,7 @@ bool trans_map_1d_amr(const dccrg::Dccrg& // For targets we need the local cells, plus a padding of 1 cell at both ends phiprof::start("computeSpatialTargetCellsForPencils"); std::vector targetCells(pencils.sumOfLengths + pencils.N * 2 * nTargetNeighborsPerPencil ); - computeSpatialTargetCellsForPencils(mpiGrid, pencils, dimension, targetCells.data()); + computeSpatialTargetCellsForPencilsWithFaces(mpiGrid, pencils, dimension, targetCells.data()); phiprof::stop("computeSpatialTargetCellsForPencils"); @@ -1165,7 +1309,7 @@ bool trans_map_1d_amr(const dccrg::Dccrg& int t1 = phiprof::initializeTimer("mapping"); int t2 = phiprof::initializeTimer("store"); -#pragma omp parallel + #pragma omp parallel { // declarations for variables needed by the threads std::vector> targetBlockData((pencils.sumOfLengths + 2 * nTargetNeighborsPerPencil * pencils.N) * WID3); @@ -1192,8 +1336,7 @@ bool trans_map_1d_amr(const dccrg::Dccrg& // Compute spatial neighbors for the source cells of the pencil. In // source cells we have a wider stencil and take into account boundaries. std::vector sourceCells(sourceLength); - // std::vector sourceCellIds(sourceLength); - computeSpatialSourceCellsForPencil(mpiGrid, pencils, pencili, dimension, sourceCells.data()/*, sourceCellIds*/); + computeSpatialSourceCellsForPencil(mpiGrid, pencils, pencili, dimension, sourceCells.data()); pencilSourceCells.push_back(sourceCells); // dz is the cell size in the direction of the pencil @@ -1205,7 +1348,7 @@ bool trans_map_1d_amr(const dccrg::Dccrg& } // Loop over velocity space blocks. Thread this loop (over vspace blocks) with OpenMP. -#pragma omp for schedule(guided) + #pragma omp for schedule(guided) for(uint blocki = 0; blocki < unionOfBlocks.size(); blocki++) { // Get global id of the velocity block @@ -1246,8 +1389,9 @@ bool trans_map_1d_amr(const dccrg::Dccrg& // Unpack the vector data Realf vector[VECL]; - pencilSourceVecData[pencili][i_trans_ps_blockv_pencil(planeVector, k, icell - 1, L)].store(vector); - + //pencilSourceVecData[pencili][i_trans_ps_blockv_pencil(planeVector, k, icell - 1, L)].store(vector); + pencilTargetValues[pencili][i_trans_pt_blockv(planeVector, k, icell - 1)].store(vector); + // Loop over 3rd (vectorized) vspace dimension for (uint iv = 0; iv < VECL; iv++) { @@ -1359,16 +1503,24 @@ int get_sibling_index(dccrg::Dccrg& mpiGr return NO_SIBLINGS; } + //CellID parent = mpiGrid.mapping.get_parent(cellid); CellID parent = mpiGrid.get_parent(cellid); if (parent == INVALID_CELLID) { - return ERROR; + std::cerr<<"Invalid parent id"< siblingarr = mpiGrid.mapping.get_all_children(parent); + // vector siblings(siblingarr.begin(), siblingarr.end()); vector siblings = mpiGrid.get_all_children(parent); auto location = std::find(siblings.begin(),siblings.end(),cellid); auto index = std::distance(siblings.begin(), location); - + if (index>7) { + std::cerr<<"Invalid parent id"<neighbor_block_data.at(i) = ccell->get_data(popID); + ccell->neighbor_block_data[i] = ccell->get_data(popID); ccell->neighbor_number_of_blocks[i] = 0; } } @@ -1453,7 +1605,7 @@ void update_remote_mapping_contribution_amr( if(ccell) { // Initialize number of blocks to 0 and neighbor block data pointer to the local block data pointer for (uint i = 0; i < MAX_NEIGHBORS_PER_DIM; ++i) { - ccell->neighbor_block_data.at(i) = ccell->get_data(popID); + ccell->neighbor_block_data[i] = ccell->get_data(popID); ccell->neighbor_number_of_blocks[i] = 0; } } @@ -1593,6 +1745,8 @@ void update_remote_mapping_contribution_amr( recvIndex = mySiblingIndex; + // std::array siblingarr = mpiGrid.mapping.get_all_children(mpiGrid.mapping.get_parent(c)); + // vector mySiblings(siblingarr.begin(), siblingarr.end()); auto mySiblings = mpiGrid.get_all_children(mpiGrid.get_parent(c)); auto myIndices = mpiGrid.mapping.get_indices(c); diff --git a/vlasovsolver/cpu_trans_map_amr.hpp b/vlasovsolver/cpu_trans_map_amr.hpp index e89112df0..07656979a 100644 --- a/vlasovsolver/cpu_trans_map_amr.hpp +++ b/vlasovsolver/cpu_trans_map_amr.hpp @@ -78,7 +78,7 @@ struct setOfPencils { std::vector getIds(const uint pencilId) const { - if (pencilId > N) { + if (pencilId >= N) { std::vector idsEmpty; return idsEmpty; } @@ -100,7 +100,8 @@ struct setOfPencils { // Find paths that members of this pencil may have in other pencils (can happen) // so that we don't add duplicates. std::vector existingSteps; - + +#pragma omp parallel for for (uint theirPencilId = 0; theirPencilId < this->N; ++theirPencilId) { if(theirPencilId == myPencilId) continue; auto theirIds = this->getIds(theirPencilId); @@ -119,7 +120,10 @@ struct setOfPencils { if(samePath) { uint theirStep = theirPath.at(myPath.size()); - existingSteps.push_back(theirStep); +#pragma omp critical + { + existingSteps.push_back(theirStep); + } } } } diff --git a/vlasovsolver/vectorclass_fallback.h b/vlasovsolver/vectorclass_fallback.h index 0df5db517..929fea644 100644 --- a/vlasovsolver/vectorclass_fallback.h +++ b/vlasovsolver/vectorclass_fallback.h @@ -28,14 +28,16 @@ */ -// Prefetching does nothing in the fallback vectorclass. +// Prefetching does nothing in the fallback vectorclass, if no system implementation +// is available +#ifndef _mm_prefetch #define _mm_prefetch(...) +#endif template class Vec4Simple { public: - const int len = 4; T val[4] __attribute__((aligned(32))); // donot initi v Vec4Simple() { } @@ -228,10 +230,10 @@ template static inline Vec4Simple operator * (const S &l,const Vec4Simple &r) { return Vec4Simple( - r.val[0]*l, - r.val[1]*l, - r.val[2]*l, - r.val[3]*l + l*r.val[0], + l*r.val[1], + l*r.val[2], + l*r.val[3] ); } @@ -260,13 +262,13 @@ static inline Vec4Simple operator / (const Vec4Simple &l, const S &r) } template -static inline Vec4Simple operator / (const S &r, const Vec4Simple &l ) +static inline Vec4Simple operator / (const S &l, const Vec4Simple &r ) { return Vec4Simple( - r/l.val[0], - r/l.val[1], - r/l.val[2], - r/l.val[3] + l/r.val[0], + l/r.val[1], + l/r.val[2], + l/r.val[3] ); } @@ -938,6 +940,21 @@ static inline Vec8Simple operator / (const Vec8Simple &l, const S &r) ); } +template +static inline Vec8Simple operator / (const S &l, const Vec8Simple &r) +{ + return Vec8Simple( + l/r.val[0], + l/r.val[1], + l/r.val[2], + l/r.val[3], + l/r.val[4], + l/r.val[5], + l/r.val[6], + l/r.val[7] + ); +} + template static inline Vec8Simple & operator += (Vec8Simple &l, const Vec8Simple &r){ l=l+r; @@ -1015,17 +1032,61 @@ template static inline Vec8Simple operator == (const Vec8Simple &l, const S &r) { return Vec8Simple( - l.val[0] == S, - l.val[1] == S, - l.val[2] == S, - l.val[3] == S, - l.val[4] == S, - l.val[5] == S, - l.val[6] == S, - l.val[7] == S + l.val[0] == r, + l.val[1] == r, + l.val[2] == r, + l.val[3] == r, + l.val[4] == r, + l.val[5] == r, + l.val[6] == r, + l.val[7] == r + ); +} + +template +static inline Vec8Simple operator != (const Vec8Simple &l, const Vec8Simple &r) +{ + return Vec8Simple( + l.val[0] != r.val[0], + l.val[1] != r.val[1], + l.val[2] != r.val[2], + l.val[3] != r.val[3], + l.val[4] != r.val[4], + l.val[5] != r.val[5], + l.val[6] != r.val[6], + l.val[7] != r.val[7] + ); +} + +template +static inline Vec8Simple operator != (const Vec8Simple &l, const S &r) +{ + return Vec8Simple( + l.val[0] != r, + l.val[1] != r, + l.val[2] != r, + l.val[3] != r, + l.val[4] != r, + l.val[5] != r, + l.val[6] != r, + l.val[7] != r ); } +template +static inline Vec8Simple operator ! (const Vec8Simple &l) +{ + return Vec8Simple( + !l.val[0], + !l.val[1], + !l.val[2], + !l.val[3], + !l.val[4], + !l.val[5], + !l.val[6], + !l.val[7] + ); +} template static inline Vec8Simple operator > (const Vec8Simple &l, const Vec8Simple &r) @@ -1231,10 +1292,6 @@ template ); } - - - - template static inline Vec8Simple min(Vec8Simple const & l, Vec8Simple const & r){ return Vec8Simple( diff --git a/vlasovsolver/vlasovmover.cpp b/vlasovsolver/vlasovmover.cpp index bae0fba32..b231f0335 100644 --- a/vlasovsolver/vlasovmover.cpp +++ b/vlasovsolver/vlasovmover.cpp @@ -85,10 +85,10 @@ void calculateSpatialTranslation( int myRank; MPI_Comm_rank(MPI_COMM_WORLD,&myRank); -// int bt=phiprof::initializeTimer("barrier-trans-pre-z","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + // int bt=phiprof::initializeTimer("barrier-trans-pre-z","Barriers","MPI"); + // phiprof::start(bt); + // MPI_Barrier(MPI_COMM_WORLD); + // phiprof::stop(bt); // ------------- SLICE - map dist function in Z --------------- // if(P::zcells_ini > 1){ @@ -98,10 +98,10 @@ void calculateSpatialTranslation( mpiGrid.update_copies_of_remote_neighbors(VLASOV_SOLVER_Z_NEIGHBORHOOD_ID); phiprof::stop(trans_timer); -// bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-z","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + // bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-z","Barriers","MPI"); + // phiprof::start(bt); + // MPI_Barrier(MPI_COMM_WORLD); + // phiprof::stop(bt); t1 = MPI_Wtime(); phiprof::start("compute-mapping-z"); @@ -113,10 +113,10 @@ void calculateSpatialTranslation( phiprof::stop("compute-mapping-z"); time += MPI_Wtime() - t1; -// bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-z","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + // bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-z","Barriers","MPI"); + // phiprof::start(bt); + // MPI_Barrier(MPI_COMM_WORLD); + // phiprof::stop(bt); trans_timer=phiprof::initializeTimer("update_remote-z","MPI"); phiprof::start("update_remote-z"); @@ -131,10 +131,10 @@ void calculateSpatialTranslation( } -// bt=phiprof::initializeTimer("barrier-trans-pre-x","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + // bt=phiprof::initializeTimer("barrier-trans-pre-x","Barriers","MPI"); + // phiprof::start(bt); + // MPI_Barrier(MPI_COMM_WORLD); + // phiprof::stop(bt); // ------------- SLICE - map dist function in X --------------- // if(P::xcells_ini > 1){ @@ -147,10 +147,10 @@ void calculateSpatialTranslation( mpiGrid.update_copies_of_remote_neighbors(VLASOV_SOLVER_X_NEIGHBORHOOD_ID); phiprof::stop(trans_timer); -// bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-x","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + // bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-x","Barriers","MPI"); + // phiprof::start(bt); + // MPI_Barrier(MPI_COMM_WORLD); + // phiprof::stop(bt); t1 = MPI_Wtime(); phiprof::start("compute-mapping-x"); @@ -162,10 +162,10 @@ void calculateSpatialTranslation( phiprof::stop("compute-mapping-x"); time += MPI_Wtime() - t1; -// bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-x","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + // bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-x","Barriers","MPI"); + // phiprof::start(bt); + // MPI_Barrier(MPI_COMM_WORLD); + // phiprof::stop(bt); trans_timer=phiprof::initializeTimer("update_remote-x","MPI"); phiprof::start("update_remote-x"); @@ -180,10 +180,10 @@ void calculateSpatialTranslation( } -// bt=phiprof::initializeTimer("barrier-trans-pre-y","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + // bt=phiprof::initializeTimer("barrier-trans-pre-y","Barriers","MPI"); + // phiprof::start(bt); + // MPI_Barrier(MPI_COMM_WORLD); + // phiprof::stop(bt); // ------------- SLICE - map dist function in Y --------------- // if(P::ycells_ini > 1) { @@ -196,10 +196,10 @@ void calculateSpatialTranslation( mpiGrid.update_copies_of_remote_neighbors(VLASOV_SOLVER_Y_NEIGHBORHOOD_ID); phiprof::stop(trans_timer); -// bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-y","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + // bt=phiprof::initializeTimer("barrier-trans-pre-trans_map_1d-y","Barriers","MPI"); + // phiprof::start(bt); + // MPI_Barrier(MPI_COMM_WORLD); + // phiprof::stop(bt); t1 = MPI_Wtime(); phiprof::start("compute-mapping-y"); @@ -211,10 +211,10 @@ void calculateSpatialTranslation( phiprof::stop("compute-mapping-y"); time += MPI_Wtime() - t1; -// bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-y","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + // bt=phiprof::initializeTimer("barrier-trans-pre-update_remote-y","Barriers","MPI"); + // phiprof::start(bt); + // MPI_Barrier(MPI_COMM_WORLD); + // phiprof::stop(bt); trans_timer=phiprof::initializeTimer("update_remote-y","MPI"); phiprof::start("update_remote-y"); @@ -229,10 +229,10 @@ void calculateSpatialTranslation( } -// bt=phiprof::initializeTimer("barrier-trans-post-trans","Barriers","MPI"); -// phiprof::start(bt); -// MPI_Barrier(MPI_COMM_WORLD); -// phiprof::stop(bt); + // bt=phiprof::initializeTimer("barrier-trans-post-trans","Barriers","MPI"); + // phiprof::start(bt); + // MPI_Barrier(MPI_COMM_WORLD); + // phiprof::stop(bt); // MPI_Barrier(MPI_COMM_WORLD); // bailout(true, "", __FILE__, __LINE__); @@ -390,12 +390,13 @@ void calculateAcceleration(const uint popID,const uint globalMaxSubcycles,const //spatial block neighbors as much in sync as possible for //adjust blocks. Real subcycleDt; - if( (step + 1) * maxVdt > dt) { - subcycleDt = max(dt - step * maxVdt, 0.0); + if( (step + 1) * maxVdt > fabs(dt)) { + subcycleDt = max(fabs(dt) - step * maxVdt, 0.0); } else{ subcycleDt = maxVdt; } - + if (dt<0) subcycleDt = -subcycleDt; + //generate pseudo-random order which is always the same irrespective of parallelization, restarts, etc. char rngStateBuffer[256]; random_data rngDataBuffer;