diff --git a/PostProc/rcat_noresm.sh b/PostProc/rcat_noresm.sh index fa9432d..9c14740 100755 --- a/PostProc/rcat_noresm.sh +++ b/PostProc/rcat_noresm.sh @@ -46,44 +46,49 @@ DELETE=0 DRYRUN="no" ERRCODE=0 ERRMSG="" +KEEP_MONTHLY="no" # Do not concatenate monthly files into yearly files MOVEDIR="/scratch/${USER}/SOURCE_FILES_TO_BE_DELETED" MOVE=0 MERGETYPE="yearly" # or "monthly" or "mergeall" or "compressonly" declare -i NTHREADS=4 POSITIONAL=() -PROGREPORTINT=5000 +PROGREPINT=1000 +USEICEFILENAMES="yes" UNITTESTMODE="no" # Used for unit testing, skip input checking, error checks, and runs -declare -i VERBOSE=0 # Use --verbose to get more output - -## Error codes -SUCCESS=0 # Routine ran without error -ERR_BADARG=02 # Bad command line argument -ERR_NCKS_MDATA=03 # Error extracting file metadata with ncks -ERR_UNSUPPORT_CAL=04 # Unsupported NetCDF calendar type -ERR_UNSUPPORT_TIME=05 # Unsupported NetCDF time units -ERR_BAD_YEAR0=06 # Error extracting year0 from calendar -ERR_BADARG_GT=07 # Bad argument(s) to greater_than -ERR_BADARG_LT=08 # Bad argument(s) to less_than -ERR_BAD_DATESTR=09 # Bad date string from ncks -ERR_BADYEAR=10 # Bad year found in history file -ERR_MULTYEARS=11 # Multiple years found in history file -ERR_MULTMONTHS=12 # Multiple months found in history file -ERR_INTERNAL=13 # Internal error (should not happen) -ERR_MISSING_FILE=14 # File does not exist -ERR_UNSUPPORT_MERGE=15 # Unsupported merge type for file -ERR_BAD_MERGETYPE=16 # Bad (unknown) merge type -ERR_BAD_COMPTYPE=17 # Unknown component type -ERR_BAD_TIME=18 # Error extracting time using ncks -ERR_EXTRACT=19 # Error extracting frame using ncks -ERR_CPRNC=20 # Error running cprnc -ERR_NOCOMPRESS=21 # No files to compress -ERR_NCRCAT=22 # Error running ncrcat -ERR_COMPARE=23 # Error comparing original and compressed data -ERR_INTERRUPT=24 # User or system interrupt +declare -i VERBOSE=0 # Use --verbose to get more output + +## Error codes. 0 and 1 are reserved for true and false Bash primitives +SUCCESS=02 # Routine ran without error +ERR_BADARG=03 # Bad command line argument +ERR_BADARG_GT=04 # Bad argument(s) to greater_than +ERR_BADARG_LT=05 # Bad argument(s) to less_than +ERR_BADYEAR=06 # Bad year found in history file +ERR_BAD_COMPTYPE=07 # Unknown component type +ERR_BAD_DATESTR=08 # Bad date string from ncks +ERR_BAD_MERGETYPE=09 # Bad (unknown) merge type +ERR_BAD_TIME=10 # Error extracting time using ncks +ERR_BAD_YEAR0=11 # Error extracting year0 from calendar +ERR_CHECKSUM=12 # Error creating a checksum +ERR_COMPARE=13 # Error comparing original and compressed data +ERR_COPY=14 # Error copying file +ERR_CPRNC=15 # Error running cprnc +ERR_EXTRACT=16 # Error extracting frame using ncks +ERR_INTERNAL=17 # Internal error (should not happen) +ERR_INTERRUPT=18 # User or system interrupt +ERR_MISSING_FILE=19 # File does not exist +ERR_MULTMONTHS=20 # Multiple months found in history file +ERR_MULTYEARS=21 # Multiple years found in history file +ERR_NCKS_MDATA=22 # Error extracting file metadata with ncks +ERR_NCRCAT=23 # Error running ncrcat +ERR_NOACCESS=24 +ERR_NOCOMPRESS=25 # No files to compress +ERR_UNSUPPORT_CAL=26 # Unsupported NetCDF calendar type +ERR_UNSUPPORT_MERGE=27 # Unsupported merge type for file +ERR_UNSUPPORT_TIME=28 # Unsupported NetCDF time units ## Keep track of failures -declare -A fail_report=() -declare -A job_status=() # Status = created, in compress, compressed, in check, pass, fail +declare -A fail_count=() # Count of job failures +declare -A job_status=() # Status = created, in compress, compressed, in check, in copy, copied, pass, fail, ERROR declare -A error_reports=() ## Have a global logfile that can be of use even in the case of an error exit declare logfilename="" @@ -93,6 +98,11 @@ declare err_lockfilename="" declare JOBLID ## Only report the job status once declare job_report_done="no" +## Path to NCO tools +declare ncrcat +declare ncks +## Top directory of output +declare outpath help() { echo -e "${tool}, version ${VERSION}\n" @@ -103,27 +113,41 @@ help() { echo " -y --year --yearly merge per year (default)" echo " -m --month --monthly merge per month" echo " -a --merge-all merge all files" -# echo " --compress-only compress files but do not merge" + echo " --compress-only compress files but do not merge" + echo " --keep-monthly do not concatenate monthly files into" + echo " yearly files" echo " -c --compress N compression strength 1-9 (default 2)" echo " -t --threads N parallel run (default 4)" echo " --compare