From f698f8126f29e4d512add2bc5710e765192e6988 Mon Sep 17 00:00:00 2001 From: Santiago Bravo Date: Mon, 9 Dec 2024 16:03:18 -0300 Subject: [PATCH] Removed shell equals true from the reported files --- main.py | 191 +++++++++------------ src/download/download_copernicus_parser.py | 11 +- src/download/download_era5_parser.py | 9 +- src/plot/plot_mdk3.py | 4 +- src/preprocessing/preprocessing_mdk3.py | 19 +- 5 files changed, 108 insertions(+), 126 deletions(-) diff --git a/main.py b/main.py index 8a44b16..96b0c1d 100644 --- a/main.py +++ b/main.py @@ -10,7 +10,7 @@ import pandas as pd from datetime import datetime from argparse import ArgumentParser -from glob import glob +from glob import glob as gg # Import medslik modules from src.utils import Utils, Config, read_oilbase @@ -183,16 +183,13 @@ def data_download_medslik( password=copernicus_pass, ) - subprocess.run( - [ - f'cp {output_path}*{identifier}*{config["simulation"]["name"]}*.nc {root_directory}/oce_files/' - ], - shell=True, - ) - subprocess.run( - [f'rm {output_path}*{identifier}*{config["simulation"]["name"]}*.nc'], - shell=True, - ) + source_files = gg(f"{output_path}*{identifier}*{config['simulation']['name']}*.nc") + destination = os.path.join(root_directory, "oce_files") + for file in source_files: + shutil.copy(file, destination) + + for file in gg(f"{output_path}*{identifier}*{config['simulation']['name']}*.nc"): + os.remove(file) if config["download"]["download_wind"]: # ensuring .cdsapirc is created in the home directory @@ -216,16 +213,13 @@ def data_download_medslik( ) process_era5(output_path=output_path, output_name=output_name) - subprocess.run( - [ - f'cp {output_path}*{identifier}*{config["simulation"]["name"]}*.nc {root_directory}/met_files/' - ], - shell=True, - ) - subprocess.run( - [f'rm {output_path}*{identifier}*{config["simulation"]["name"]}*.nc'], - shell=True, - ) + source_files = gg(f"{output_path}*{identifier}*{config['simulation']['name']}*.nc") + destination = os.path.join(root_directory, "met_files") + for file in source_files: + shutil.copy(file, destination) + + for file in gg(f"{output_path}*{identifier}*{config['simulation']['name']}*.nc"): + os.remove(file) def run_preproc(config: dict, exp_folder: str, lon_min, lon_max, lat_min, lat_max): """ @@ -343,98 +337,75 @@ def run_medslik_sim(self, simdir, simname, separate_slicks=False): output_dir = f"{model_dir}OUT/MDK_SIM_{year}_{month:02d}_{day:02d}_{hour:02d}{minute:02d}_{simname}/." - # removing old outputes just to be sure - subprocess.run([f"rm -rf {output_dir}"], shell=True) - - if separate_slicks == False: - # copy METOCEAN files to MEDSLIK-II installation - subprocess.run( - [f"cp {simdir}{simname}/oce_files/*.mrc {model_dir}RUN/TEMP/OCE/"], - shell=True, - check=True, - ) - subprocess.run( - [f"cp {simdir}{simname}/met_files/*.eri {model_dir}RUN/TEMP/MET/"], - shell=True, - check=True, - ) - # copy bnc files - subprocess.run( - [f"cp {simdir}{simname}/bnc_files/* {model_dir}DTM_INP/"], - shell=True, - check=True, - ) - # copy Extract and config files - subprocess.run( - [ - f"cp {simdir}{simname}/xp_files/medslik_II.for {model_dir}RUN/MODEL_SRC/" - ], - shell=True, - check=True, - ) - subprocess.run( - [f"cp {simdir}{simname}/xp_files/config2.txt {model_dir}RUN/"], - shell=True, - check=True, - ) - subprocess.run( - [f"cp {simdir}{simname}/xp_files/config1.txt {model_dir}RUN/"], - shell=True, - check=True, - ) - # Compile and start running - subprocess.run( - [f"cd {model_dir}RUN/; sh MODEL_SRC/compile.sh; ./RUN.sh"], - shell=True, - check=True, - ) + # Remove old outputs (equivalent to `rm -rf`) + if os.path.exists(output_dir): + shutil.rmtree(output_dir) + + if not separate_slicks: + # Copy METOCEAN files + oce_files = gg(f"{simdir}{simname}/oce_files/*.mrc") + met_files = gg(f"{simdir}{simname}/met_files/*.eri") + bnc_files = gg(f"{simdir}{simname}/bnc_files/*") + xp_files = { + "medslik_II.for": os.path.join(simdir, simname, "xp_files", "medslik_II.for"), + "config2.txt": os.path.join(simdir, simname, "xp_files", "config2.txt"), + "config1.txt": os.path.join(simdir, simname, "xp_files", "config1.txt"), + } + + # Copy METOCEAN, MET, and BNC files + for file in oce_files: + shutil.copy(file, os.path.join(model_dir, "RUN", "TEMP", "OCE")) + for file in met_files: + shutil.copy(file, os.path.join(model_dir, "RUN", "TEMP", "MET")) + for file in bnc_files: + shutil.copy(file, os.path.join(model_dir, "DTM_INP")) + + # Copy other required files + for dest, src in xp_files.items(): + shutil.copy(src, os.path.join(model_dir, "RUN", dest if "config" in dest else "MODEL_SRC")) + + # Compile and start running (replacing `cd` with `cwd`) + compile_script = "MODEL_SRC/compile.sh" + run_script = "RUN.sh" + subprocess.run(["sh", compile_script], check=True, cwd=os.path.join(model_dir, "RUN")) + subprocess.run(["./" + run_script], check=True, cwd=os.path.join(model_dir, "RUN")) else: - slicks = glob(f"{simdir}{simname}/xp_files/*/") - for i in range(0, len(slicks)): - subprocess.run( - [f"cp {simdir}{simname}/oce_files/*.mrc {model_dir}RUN/TEMP/OCE/"], - shell=True, - ) - subprocess.run( - [f"cp {simdir}{simname}/met_files/*.eri {model_dir}RUN/TEMP/MET/"], - shell=True, - ) - # copy bnc files - subprocess.run( - [f"cp {simdir}{simname}/bnc_files/* {model_dir}DTM_INP/"], - shell=True, - ) - # copy Extract and config files - subprocess.run( - [ - f"cp {simdir}{simname}/xp_files/medslik_II.for {model_dir}RUN/MODEL_SRC/" - ], - shell=True, - ) - subprocess.run( - [f"cp {simdir}{simname}/xp_files/config2.txt {model_dir}RUN/"], - shell=True, - ) - subprocess.run( - [ - f"cp {simdir}{simname}/xp_files/slick{i+1}/config1.txt {model_dir}RUN/" - ], - shell=True, - ) - # Compile and start running - subprocess.run( - [f"cd {model_dir}RUN/; sh MODEL_SRC/compile.sh; ./RUN.sh"], - shell=True, - check=True, - ) + # Handle separate slicks + slicks = gg(f"{simdir}{simname}/xp_files/*/") + for i, slick_dir in enumerate(slicks): + # Copy METOCEAN, MET, and BNC files + oce_files = gg(f"{simdir}{simname}/oce_files/*.mrc") + met_files = gg(f"{simdir}{simname}/met_files/*.eri") + bnc_files = gg(f"{simdir}{simname}/bnc_files/*") + config1_path = os.path.join(simdir, simname, f"xp_files/slick{i + 1}/config1.txt") + + for file in oce_files: + shutil.copy(file, os.path.join(model_dir, "RUN", "TEMP", "OCE")) + for file in met_files: + shutil.copy(file, os.path.join(model_dir, "RUN", "TEMP", "MET")) + for file in bnc_files: + shutil.copy(file, os.path.join(model_dir, "DTM_INP")) + shutil.copy(config1_path, os.path.join(model_dir, "RUN", "config1.txt")) - # Send files to case dir and remove temp files - subprocess.run([f"cp -r {output_dir} {simdir}{simname}/out_files/"], shell=True) - subprocess.run( - [f"rm -rf {simdir}{simname}/out_files/MET {simdir}{simname}/out_files/OCE"], - shell=True, - ) + # Compile and start running + compile_script = "MODEL_SRC/compile.sh" + run_script = "RUN.sh" + subprocess.run(["sh", compile_script], check=True, cwd=os.path.join(model_dir, "RUN")) + subprocess.run(["./" + run_script], check=True, cwd=os.path.join(model_dir, "RUN")) + + # Copy output files (replacing `cp -r`) + output_dest = os.path.join(simdir, simname, "out_files") + if os.path.exists(output_dir): + shutil.copytree(output_dir, output_dest, dirs_exist_ok=True) + + # Remove temporary MET and OCE files (equivalent to `rm -rf`) + temp_met = os.path.join(output_dest, "MET") + temp_oce = os.path.join(output_dest, "OCE") + if os.path.exists(temp_met): + shutil.rmtree(temp_met) + if os.path.exists(temp_oce): + shutil.rmtree(temp_oce) if __name__ == "__main__": diff --git a/src/download/download_copernicus_parser.py b/src/download/download_copernicus_parser.py index 07eedf7..0aeb9ff 100644 --- a/src/download/download_copernicus_parser.py +++ b/src/download/download_copernicus_parser.py @@ -74,7 +74,9 @@ def download_copernicus( ds.to_netcdf(output_name) # remove the temporary files - subprocess.run([f"rm -rf {output_path}temp.nc"], shell=True) + temp_file = os.path.join(output_path, "temp.nc") + if os.path.exists(temp_file): + os.remove(temp_file) else: @@ -150,6 +152,7 @@ def download_copernicus( ds.to_netcdf(output_name) # remove the temporary files - subprocess.run( - [f"rm -rf {output_path}/curr.nc {output_path}/temp.nc"], shell=True - ) + temp_files = [os.path.join(output_path, "curr.nc"), os.path.join(output_path, "temp.nc")] + for temp_file in temp_files: + if os.path.exists(temp_file): + os.remove(temp_file) diff --git a/src/download/download_era5_parser.py b/src/download/download_era5_parser.py index b23c1ef..01f7d01 100644 --- a/src/download/download_era5_parser.py +++ b/src/download/download_era5_parser.py @@ -6,6 +6,7 @@ import argparse import subprocess import time +from glob import glob as gg # Functions outside this script from WITOIL_iMagine.src.utils.utils import * @@ -79,7 +80,10 @@ def process_era5(output_path,output_name): met.to_netcdf(output_name) #remove the temporary files - subprocess.run([f'rm -rf {output_path}/temp*.nc'],shell=True) + temp_files = gg(os.path.join(output_path, "temp*.nc")) + for temp_file in temp_files: + if os.path.exists(temp_file): + os.remove(temp_file) if __name__ == '__main__': @@ -125,5 +129,4 @@ def process_era5(output_path,output_name): # os.system('rm ' + out_folder + '/output.nc') - # os.system('rm ' + out_folder + '/pre_*.nc') - + # os.system('rm ' + out_folder + '/pre_*.nc') \ No newline at end of file diff --git a/src/plot/plot_mdk3.py b/src/plot/plot_mdk3.py index f897f30..f8f5bfa 100644 --- a/src/plot/plot_mdk3.py +++ b/src/plot/plot_mdk3.py @@ -180,7 +180,7 @@ def create_gif(self): f"magick -delay 20 -loop 0 {path}/*surf_oil_*.png \ {self.out_figures}/oil_concentration_{self.config['simulation']['name']}.gif" ], - shell=True, + # shell=True, ) def plot_pyngl( @@ -208,7 +208,7 @@ def plot_pyngl( {sim_length} {plot_lon[0]} {plot_lon[1]} \ {plot_lat[0]} {plot_lat[1]}" ], - shell=True, + # shell=True, check=True, ) diff --git a/src/preprocessing/preprocessing_mdk3.py b/src/preprocessing/preprocessing_mdk3.py index 425692f..f9c15c2 100644 --- a/src/preprocessing/preprocessing_mdk3.py +++ b/src/preprocessing/preprocessing_mdk3.py @@ -219,14 +219,20 @@ def process_medslik_memmory_array(self): med_for = f'{self.exp_folder}/xp_files/medslik_II.for' - subprocess.run([f'cp WITOIL_iMagine/src/templates/medslik_II_template.for {med_for}'],shell=True) + # Refactored: Copying the template file + template_path = 'WITOIL_iMagine/src/templates/medslik_II_template.for' + shutil.copy(template_path, med_for) - # Replacing NMAX in medslik fortran with a python function + # Replacing NMAX in medslik Fortran with a Python function Utils.search_and_replace(med_for, 'NMAX', str(nmax)) def configuration_parameters(self): - subprocess.run([f'cp WITOIL_iMagine/src/templates/config2.txt {self.exp_folder}/xp_files/config2.txt'],shell=True) + source_file = 'WITOIL_iMagine/src/templates/config2.txt' + destination_file = f'{self.exp_folder}/xp_files/config2.txt' + + # Refactored: Copy the file + shutil.copy(source_file, destination_file) def common_grid(self): @@ -271,9 +277,9 @@ def write_config_files(self, config_file = f"WITOIL_iMagine/cases/{simname}/xp_files/config1.txt" else: config_file = f"WITOIL_iMagine/cases/{simname}/xp_files/slick{s_num+1}/config1.txt" - subprocess.run( - [f"cp WITOIL_iMagine/src/templates/config1_template_0.txt {config_file}"], shell=True - ) + # Refactored: Copy the template file + source_file = "WITOIL_iMagine/src/templates/config1_template_0.txt" + shutil.copy(source_file, config_file) # adding spill Name - Add slick number if separate slicks if separate_slicks == False: Utils.search_and_replace(config_file, "RUNNAME", simname) @@ -331,4 +337,3 @@ def write_config_files(self, if __name__ == "__main__": pass -