Fix shock1d meshmode meshgen, add profiling control, skip scalability…

… meshgen.

scripts/multi_scalability.sh

@@ -144,59 +144,24 @@ cd ${DRIVER_PATH}/${test_path}
 nrank=${NUM_PROCS_1}
 while [ $nrank -le $NUM_PROCS ]; do
 
-    if [[ "${nrank}" == "1" ]]; then
-        msize="48"
-        nelem="24036"
-    elif [[ "${nrank}" == "2" ]]; then
-        msize="30.5"
-        nelem="47908"
-    elif [[ "${nrank}" == "4" ]]; then
-        msize="21.5"
-        nelem="96425"
-    elif [[ "${nrank}" == "8" ]]; then
-        msize="16.05"
-        nelem="192658"
-    elif [[ "${nrank}" == "16" ]]; then
-        msize="12.3"
-        nelem="383014"
-    elif [[ "${nrank}" == "32" ]]; then
-        msize="9.535"
-        nelem="768891"
-    elif [[ "${nrank}" == "64" ]]; then
-        msize="7.45"
-        nelem="1536571"
-    elif [[ "${nrank}" == "128" ]]; then
-        msize="5.8545"
-        nelem="3071835"
-    elif [[ "${nrank}" == "256" ]]; then
-        msize="4.61"
-        nelem="6142150"
-    elif [[ "${nrank}" == "512" ]]; then
-        msize="3.6389"
-        nelem="12283032"
-    elif [[ "${nrank}" == "1024" ]]; then
-        msize="3.07"
-        nelem="24423603"
-    elif [[ "${nrank}" == "2048" ]]; then
-        msize="3.6389"
-        nelem="12283032"
-    fi
-
-
     casename="${running_casename_base}_np${nrank}"
     printf "** Running ${casename} on ${nrank} ranks.\n"
 
-    cd data
-    rm actii.msh
-    ./mkmsh --size=${msize} --nelem=${nelem} --link
-    cd ../
+    # Disable mesh generation, for now
+    # cd data
+    # rm actii.msh
+    # ./mkmsh --size=${msize} --nelem=${nelem} --link
+    # cd ../
+    #
+    # Create nranks-specific yaml input file
     rm -f run_params_np${nrank}.yaml
     sed "s|mesh_filename: .*|mesh_filename: data/actii_np${nrank}.msh|" run_params.yaml > run_params_np${nrank}.yaml
     set -x
     runoptions="-n ${nrank}"
     if [[ ! -z ${NUM_NODES} ]];then
         runoptions="-N ${NUM_NODES} -n ${nrank}"
     fi
+    # Run prediction driver with generated yaml file
     $MPI_EXEC ${runoptions} $PARALLEL_SPAWNER python -u -O -m mpi4py driver.py -c ${casename} -g ${LOG_PATH} -i run_params_np${nrank}.yaml --log --lazy
     return_code=$?
     set +x

y3prediction/prediction.py

@@ -1115,6 +1115,12 @@ def main(actx_class, restart_filename=None, target_filename=None,
     rank = comm.Get_rank()
     nparts = comm.Get_size()
 
+    first_profiling_step = 4
+    last_profiling_step = 104
+
+    MPI.Pcontrol(2)
+    MPI.Pcontrol(0)
+
     from mirgecom.simutil import global_reduce as _global_reduce
     global_reduce = partial(_global_reduce, comm=comm)
 
@@ -1139,6 +1145,8 @@ def main(actx_class, restart_filename=None, target_filename=None,
     from mirgecom.io import read_and_distribute_yaml_data
     input_data = read_and_distribute_yaml_data(comm, user_input_file)
 
+    use_callbacks = configurate("use_callbacks", input_data, True)
+
     use_gmsh = configurate("use_gmsh", input_data, True)
     from mirgecom.array_context import initialize_actx, actx_class_is_profiling
     use_tpe = configurate("use_tensor_product_elements", input_data, False)
@@ -3623,9 +3631,14 @@ def my_partitioner(mesh, tag_to_elements, num_ranks):
             volume_to_local_mesh_data, global_nelements = distribute_mesh(
                 comm, get_mesh_data, partition_generator_func=part_func)
 
-    local_nelements = volume_to_local_mesh_data["fluid"][0].nelements
+    fluid_nelements = volume_to_local_mesh_data["fluid"][0].nelements
+    wall_nelements = 0
+    print(f"Fluid elements: {fluid_nelements}")
     if use_wall:
-        local_nelements += volume_to_local_mesh_data["wall"][0].nelements
+        wall_nelements = volume_to_local_mesh_data["wall"][0].nelements
+        print(f"Wall elements: {wall_nelements}")
+    local_nelements = fluid_nelements + wall_nelements
+    print(f"Number of elements: {local_nelements}")
 
     # target data, used for sponge and prescribed boundary condtitions
     if target_filename:  # read the grid from restart data
@@ -6718,10 +6731,17 @@ def my_pre_step(step, t, dt, state):
             comm.Barrier()  # cross and dot t's and i's (sync point)
             raise
 
+        if step == first_profiling_step:
+            MPI.Pcontrol(2)
+            MPI.Pcontrol(1)
+
         return stepper_state.get_obj_array(), dt
 
     def my_post_step(step, t, dt, state):
 
+        if step == last_profiling_step:
+            MPI.Pcontrol(0)
+
         if step == first_step+2:
             with gc_timer:
                 import gc
@@ -7293,12 +7313,13 @@ def my_rhs(t, state):
                                   current_cfl, t_final, constant_cfl)
     """
 
+    pre_step_callback = my_pre_step if use_callbacks else None
+    post_step_callback = my_post_step if use_callbacks else None
     if advance_time:
         current_step, current_t, current_stepper_state_obj = \
             advance_state(rhs=my_rhs, timestepper=timestepper,
-                          pre_step_callback=my_pre_step,
-                          #pre_step_callback=None,
-                          post_step_callback=my_post_step,
+                          pre_step_callback=pre_step_callback,
+                          post_step_callback=post_step_callback,
                           istep=current_step, dt=current_dt,
                           t=current_t, t_final=t_final,
                           force_eval=force_eval,
@@ -7332,6 +7353,9 @@ def my_rhs(t, state):
                                              smoothness_beta=current_av_sbeta,
                                              smoothness_kappa=current_av_skappa,
                                              smoothness_d=current_av_sd)
+
+    MPI.Pcontrol(0)
+
     if use_wall:
         current_wv = current_stepper_state.wv
         current_wdv = create_wall_dependent_vars_compiled(current_wv)

y3prediction/shock1d.py

@@ -463,11 +463,8 @@ def get_mesh(dim, size, bl_ratio, interface_ratio, angle=0.,
         # this only works for non-slanty meshes
         def get_meshmode_mesh(a, b, nelements_per_axis, boundary_tag_to_face):
 
-            if use_quads:
-                from meshmode.mesh import TensorProductElementGroup
-                group_cls = TensorProductElementGroup
-            else:
-                group_cls = None
+            from meshmode.mesh import TensorProductElementGroup
+            group_cls = TensorProductElementGroup if use_quads else None
 
             mesh = generate_regular_rect_mesh(
                 a=a, b=b, nelements_per_axis=nelements_per_axis,
@@ -480,7 +477,7 @@ def get_meshmode_mesh(a, b, nelements_per_axis, boundary_tag_to_face):
             x_avg = np.sum(x, axis=1)/x.shape[1]
             tag_to_elements = {
                 "fluid": np.where(x_avg < fluid_length)[0],
-                "wall": np.where(x_avg > fluid_length)[0]}
+                "wall_insert": np.where(x_avg > fluid_length)[0]}
 
             return mesh, tag_to_elements