camera_cards

#!/usr/bin/env bash

# a script to turn raw camera cards into SIPs

# set script variables
DEPENDENCIES=(ffmpeg md5sum tree mediaconch exiftool ffprobe)
OP="${USER}"
AIP="YES"
TAR="NO"

# define functions
_cleanup(){
    _report -rt "Process cancelled" # tell operator that script is ending
    exit 1
}

_generate_metadata(){
    _report -g "Generating metadata reports..."
    # create tree of all files in package
    TREE="${METADATA_REPORTS_DIR}/${MEDIAID}_tree.txt"
    tree -DaNs --du --timefmt "%Y-%m-%dT%H:%M:%SZ" "${CAMERA_CARD_DIR[@]}" > "${TREE}"
    SAVE_FILE_INVENTORY="${METADATA_REPORTS_DIR}/${MEDIAID}_card_inventory.csv"
    cp -n "${FILE_INVENTORY}" "${SAVE_FILE_INVENTORY}"
    # create mediainfo, exiftool, ffprobe reports
    while read -r FILE ; do
        FN="$(basename "${FILE}")"
        MEDIAINFO_OUTPUT="${METADATA_REPORTS_DIR}/${FN}_mediainfo.txt"
        EXIFTOOL_OUTPUT="${METADATA_REPORTS_DIR}/${FN}_exiftool.txt"
        FFPROBE_OUTPUT="${METADATA_REPORTS_DIR}/${FN}_ffprobe.xml"
        mediaconch -mi -ft "${FILE}" >> "${MEDIAINFO_OUTPUT}"
        exiftool "${FILE}" >> "${EXIFTOOL_OUTPUT}"
        ffprobe -v 0 "${FILE}" -show_format -show_streams -show_data -show_error -show_versions -show_chapters -noprivate -of xml="q=1:x=1" > "${FFPROBE_OUTPUT}"
        if [[ ! -s "${MEDIAINFO_OUTPUT}" ]] || [[ ! -s "${EXIFTOOL_OUTPUT}" ]] || [[ ! -s "${FFPROBE_OUTPUT}" ]] ; then
            _report -r "Metadata generation process created one or more blank files. Input files may be empty. Review metadata output at ${METADATA_REPORTS_DIR}."
            _write_package_log "POSSIBLE_ERROR_REVIEW" "Metadata generation process created one or more blank files"
        fi
    done < <(_read_inventory -r "${CLIP_SELECTION}" -f ",video,\|,audio," "FILE_NAME")
    _report -g "Metadata reports generated for ${MEDIAID}"
}

_maketemp(){
    mktemp -t "$(basename "${0}")"
    if [ "${?}" -ne 0 ]; then
        _report -rt "${0}: Can't create temp file, exiting..."
        exit 1
    fi
}

_report(){
    local RED="$(tput setaf 1)"   # Red      - For Warnings
    local GREEN="$(tput setaf 2)" # Green    - For Declarations
    local BLUE="$(tput setaf 4)"  # Blue     - For Questions
    local NC="$(tput sgr0)"       # No Color
    local COLOR=""
    local STARTMESSAGE=""
    local ECHOOPT=""
    OPTIND=1
    while getopts ":bgrstn" OPT; do
        case "${OPT}" in
            b) COLOR="${BLUE}" ;;                         # question mode, use color blue
            g) COLOR="${GREEN}" ;;                        # declaration mode, use color green
            r) COLOR="${RED}" ;;                          # warning mode, use color red
            s) STARTMESSAGE+=([$(basename "${0}")] ) ;;   # prepend scriptname to the message
            t) STARTMESSAGE+=($(date +%FT%T) '- ' ) ;;    # prepend timestamp to the message
            n) ECHOOPT="-n" ;;                            # to avoid line breaks after echo
        esac
    done
    shift $(( OPTIND - 1 ))
    MESSAGE="${1}"
    echo "${ECHOOPT}" "${COLOR}${STARTMESSAGE[@]} ${MESSAGE}${NC}"
}

_usage(){
cat <<EOF
$(basename "${0}")
 1. By default, this script will detect a camera card file/directory structure and transform the original camera card files into a usable AIP. The script will concatenate video files into a single file, move important metadata files into a new directory structure, and create a log of these changes."
  Current camera cards recognized:
      - AVCHD (e.g. Canon C100)
      - Canon XF (e.g. Canon C300)
      - P2 (Panasonic)
      - XAVC (Sony)
      - XDCAM EX (e.g. Canon XF100)
2. If you prefer to package files as-is to preserve your original camera directories, you can choose to compress your directory structure into a tarball. To select this option, use the '-t' flag.
If your camera card structure is not recognized, the script will prompt you to choose one or both of these strategies. If you want to create an AIP with concatenated video files and restructured metadata directories, it is recommended that you review the output to make sure the script was able to process your camera files properly.

Your input package is the top-level directory of your camera card, which should be mounted on your computer or provided as a directory.
Your output package will be named after the MEDIAID you supply and delivered to the AIP destination set in nmaahcmmconfig (AIP_DESTINATION).

The AIP_DESTINATION can be set via nmaahcconfig, or via the -o options, or the script with request it.

Dependencies: ${DEPENDENCIES[@]}

Usage: $(basename "${0}") -m MEDIAID [ -o AIP_DESTINATION ] [-tca] /path/to/input/camera_card_directory [ /path/to/input/camera_card_directory_2... ]
  -m MEDIAID (type media id for final package, e.g. SC0001_20190101_SMITH_VHS_01)
  -N <parts> (List the numbers of the clips that should be packaged.
              Multiple clip numbers may be listed as comma-delimited ranges; for
              instance "-N '1,3-5'" would output the 1st, 3rd, 4th, and 5th clips and not others.)
  -f FORMAT (choose the output format (as a file extension) for the rewrapping process, such as 'mxf' or 'mkv'. The default is set to match the input card set. Experimental.)
  -o AIP_DESTINATION (the output directory for the package); if not supplied, script will look for it in nmaahcconfig
  -t tar camera files and folders into a tarball, compressed with gzip
  -a create packaged AIP; this is the default option, but you can use this flag to specify you want an AIP alongside a tarball (e.g. '-ta')
  -n Do not actually package anything, but just show information about the input and what would happen.
  -s SUFFIX   If unset, the default suffix of '_concatenated' will be used on outputs.
  -1          Enable single-output mode. If this is set then no metadata reports will be saved and only one concatenated media output (with no package structure) will be sent to the AIP_DESTINATION.
  -h display this help

  By default, this script will try to losslessly rewrap the contents of the camera cards from input to a concatenated output; however, setting this profile option will transcode the contents instead. Setting a profile will disable lossless verification.
  -p [xn|xp|h] where
  -p xn     creates an NSTC XDCam/PCM output in an mov container
  -p xp     creates an PAL XDCam/PCM output in an mov container
  -p h      creates an h264/aac output in an mp4 container

  Options specific to the GENERAL camera card type:
  -k [y|n]  If the camera card is GENERAL, set '-k y' to keep a copy of all files from the original camera card with the aip. Defaults to '-k n' to not keep them.

EOF
    exit
}

_write_package_log(){
    if [[ -z "${LOG}" ]] ; then
        _report -r "Error, can not write to log, ingest log not yet created"
        exit
    fi
    if [[ ! -w "${METADATADIR}" ]] ; then
        _report -r "Error, can not write to log as the metadata directory, ${METADATADIR}, is not writable."
        exit
    fi
    KEY="${1}"                              # key is first string passed to function
    VALUE="${2}"                            # value is variable passed to function
    OPTIND=1
    while getopts ":t" OPT; do
        case "${OPT}" in
            t) KEY="${2}" && VALUE="$(date +%FT%T)" ;;    # value is day and time in format YYYY-MM-DDTHH:MM:
        esac
    done
    # need to add yaml style escaping
    if [[ -n "${VALUE}" ]] ; then
        echo "${KEY}: ${VALUE}" >> "${LOG}"
    fi
}

_read_inventory(){
    _expand_list(){
        # takes a comma-seperated list of ranges and expands ranges and switches to pipe delimits
        LIST="${1}"
        unset EXPANDED
        IFS=',' read -a RANGES  <<< "${LIST}"
        for RANGE in "${RANGES[@]}"; do
            if [[ ${RANGE} == *"-"* ]] ; then
                IFS=- read START END <<< "${RANGE}"
                [ -z "${START}" ] && continue
                [ -z "${END}" ] && END="${START}"
                for (( i=START ; i <= END ; i++ )); do
                    if [[ -z "${EXPANDED}" ]] ; then
                        EXPANDED+="${i}"
                    else
                        EXPANDED+="\|${i}"
                    fi
                done
            else
                if [[ -z "${EXPANDED}" ]] ; then
                    EXPANDED+="${RANGE}"
                else
                    EXPANDED+="\|${RANGE}"
                fi
            fi
        done
        echo "${EXPANDED}"
    }
    OPTIND=1
    while getopts ":f:r:" OPT; do
        case "${OPT}" in
            f) FILTER="${OPTARG}" ;;
            r) GRAB_BY_VIDEO_ASSOCIATED_NUM="${OPTARG}" ;;
        esac
    done
    shift $(( OPTIND - 1 ))
    COL_NAME="${1}"
    if [[ -f "${FILE_INVENTORY}" ]] ; then
        if [[ -n "${GRAB_BY_VIDEO_ASSOCIATED_NUM}" ]] ; then
            FILTERED_LIST="$(tail -n +2 "${FILE_INVENTORY}" | grep  "[^,]*,[^,]*,[^,]*,[^,]*,\($(_expand_list "${GRAB_BY_VIDEO_ASSOCIATED_NUM}")\),")"
        else
            FILTERED_LIST="$(tail -n +2 "${FILE_INVENTORY}")"
        fi
        COL_NUM="$(head -n1 "${FILE_INVENTORY}" | awk -v RS=',' "/${COL_NAME}/{print NR; exit}")"
        if [[ -n "${COL_NAME}" ]] ; then
            if [[ -z "${COL_NUM}" ]] ; then
                _report -w "The _read_inventory function was called to find a column called ${COL_NAME} but it isn't found in the inventory."
            fi
            if [[ -n "${FILTER}" ]] ; then
                echo "${FILTERED_LIST}" | grep "$FILTER" | cut -d "," -f "${COL_NUM}"
            else
                echo "${FILTERED_LIST}" | cut -d "," -f "${COL_NUM}"
            fi
        else
            echo "${FILTERED_LIST}" | grep "$FILTER"
        fi
    else
        _report -w "The _read_inventory function was called but there's no inventory at ${FILE_INVENTORY}."
    fi
}

_make_inventory(){
    _catalog_video(){
        FILE="${1}"
        IFS="," read -r Z DURATION COMPANY_NAME PRODUCT_NAME MOD_TS TC < <(ffprobe -sexagesimal -v 0 "${FILE}" -show_entries format=duration:format_tags=company_name,product_name,modification_date,timecode -of csv)
        IFS="," read -r Z AUDIO_CODEC AUDIO_SAMPLE_FMT < <(ffprobe -sexagesimal -v 0 "${FILE}" -select_streams a:0 -show_entries stream=codec_name,sample_fmt -of csv | grep "^stream,")
        echo "${CARD_NUM},${CAMERA_CARD_TYPE},${VIDEO_NUM},${FILE_NUM},,video,${DURATION},${COMPANY_NAME},${PRODUCT_NAME},${MOD_TS},${TC},${FILE},${AUDIO_CODEC},${AUDIO_SAMPLE_FMT}" >> "${FILE_INVENTORY}"
    }
    _lookup_associated_video_file(){
        # lookup associated video number for any metadata file
        BASENAME="${FILE%.*}"
        _read_inventory -f "video,.*,${BASENAME}" VIDEO_NUM
    }
    echo "CARD_NUM,CARD_TYPE,VIDEO_NUM,FILE_NUM,ASSOCIATED_VIDEO_NUM,MEDIA_TYPE,DURATION,COMPANY_NAME,PRODUCT_NAME,MOD_TS,TIMECODE,FILE_NAME,AUDIO_CODEC,AUDIO_SAMPLE_FMT" > "${FILE_INVENTORY}"
    CARD_NUM=1
    VIDEO_NUM=1
    FILE_NUM=1
    for CARD in "${CAMERA_CARD_DIR[@]}" ; do
        _identify_card_type "${CARD}"
        if [[ "${CAMERA_CARD_TYPE}" == "P2" ]] ; then
            while read FILE ; do
                _catalog_video "${FILE}"
                ((VIDEO_NUM++))
                ((FILE_NUM++))
            done < <(find "${CARD}" ! -name ".DS_Store" -path "*CONTENTS/VIDEO*" -iname "*.${EXTENSION}" | sort)
            while read FILE ; do
                echo "${CARD_NUM},${CAMERA_CARD_TYPE},0,${FILE_NUM},,audio,${FILE},${AUDIO_CODEC},${AUDIO_SAMPLE_FMT}" >> "${FILE_INVENTORY}"
                ((FILE_NUM++))
            done < <(find "${CARD}" ! -name ".DS_Store" -path "*CONTENTS/AUDIO*" -iname "*.${EXTENSION}" | sort)
        elif [[ "${CAMERA_CARD_TYPE}" == "GENERAL" ]] ; then
            while read FILE ; do
                if [[ -n $(mediainfo --Inform="General;%VideoCount%" "${FILE}") ]] ; then
                    IFS="," read -r Z DURATION COMPANY_NAME PRODUCT_NAME MOD_TS TC < <(ffprobe -sexagesimal -v 0 "${FILE}" -show_entries format=duration:format_tags=company_name,product_name,modification_date,timecode -of csv)
                    echo "${CARD_NUM},${CAMERA_CARD_TYPE},${VIDEO_NUM},${FILE_NUM},,video,${DURATION},${COMPANY_NAME},${PRODUCT_NAME},${MOD_TS},${TC},${FILE},${AUDIO_CODEC},${AUDIO_SAMPLE_FMT}" >> "${FILE_INVENTORY}"
                    ((VIDEO_NUM++))
                elif [[ -n $(mediainfo --Inform="General;%AudioCount%" "${FILE}") ]] ; then
                    echo "${CARD_NUM},${CAMERA_CARD_TYPE},0,${FILE_NUM},,audio,,,,,,${FILE},${AUDIO_CODEC},${AUDIO_SAMPLE_FMT}" >> "${FILE_INVENTORY}"
                else
                    echo "${CARD_NUM},${CAMERA_CARD_TYPE},0,${FILE_NUM},$(_lookup_associated_video_file),metadata,,,,,,${FILE},," >> "${FILE_INVENTORY}"
                fi
                ((FILE_NUM++))
            done < <(find "${CARD}" ! -name ".DS_Store" -type f | sort)
        elif [[ "${CAMERA_CARD_TYPE}" == "XAVC" ]] ; then
            while read FILE ; do
                _catalog_video "${FILE}"
                ((VIDEO_NUM++))
                ((FILE_NUM++))
            done < <(find "${CARD}"! -name ".DS_Store"  -iname "*.${EXTENSION}" ! -path "*/PROAV/*" ! -path "*/Sub/*" -type f | sort)
        elif [[ "${CAMERA_CARD_TYPE}" ==  "MXF" ]] ; then
            while read FILE ; do
                _catalog_video "${FILE}"
                ((VIDEO_NUM++))
                ((FILE_NUM++))
            done < <(find "${CARD}" ! -name ".DS_Store" -iname "*.${EXTENSION}" -type f | sort)
        else
            while read FILE ; do
                _catalog_video "${FILE}"
                ((VIDEO_NUM++))
                ((FILE_NUM++))
            done < <(find "${CARD}" ! -name ".DS_Store" -iname "*.${EXTENSION}" -type f | sort)
        fi
        if [[ -n "${RSYNC_OPTIONS[@]}" ]] ; then
            while read FILE ; do
                if [[ -z "$(grep ",${FILE}," "${FILE_INVENTORY}")" ]] ; then
                    echo "${CARD_NUM},${CAMERA_CARD_TYPE},0,${FILE_NUM},$(_lookup_associated_video_file),metadata,,,,,,${FILE},," >> "${FILE_INVENTORY}"
                    ((FILE_NUM++))
                fi
            done < <(find "${CARD}" ! -name ".DS_Store" -type f "${RSYNC_OPTIONS[@]}" | sort)
        fi
        # grab everything else
        while read FILE ; do
            if [[ -z "$(grep ",${FILE}," "${FILE_INVENTORY}")" ]] ; then
                echo "${CARD_NUM},${CAMERA_CARD_TYPE},0,${FILE_NUM},$(_lookup_associated_video_file),other,,,,,,${FILE},," >> "${FILE_INVENTORY}"
                ((FILE_NUM++))
            fi
        done < <(find "${CARD}" ! -name ".DS_Store" -type f | sort)
        ((CARD_NUM++))
    done
}

_convert_hhmmssmmm2ns(){
    TS="${1}"
    echo "${TS}" | awk -F ":" '{ printf "%8i\n", (($1 * 3600) + ($2 * 60) + $3 ) * 100000}'
}

_print_table(){
    # this prints a table of files but also creates a chapter list at the same time
    CHAPTER_LIST="$(_maketemp).ffmetadata"
    echo ";FFMETADATA1" > "${CHAPTER_LIST}"
    START_VALUE=0
    END_VALUE=0
    echo
    echo "Card#   Index#  File Name                    Modification Date     Duration            Timecode"
    while IFS=, read -r CARD_NUM CARD_TYPE VIDEO_NUM FILE_NUM ASSOCIATED_VIDEO_NUM MEDIA_TYPE DURATION COMPANY_NAME PRODUCT_NAME MOD_TS TIMECODE FILE_NAME AUDIO_CODEC AUDIO_SAMPLE_FMT ; do
        FILEBASENAME="$(basename "${FILE_NAME}")"
        if [[ -z "${CLIP_SELECTION}" || "$(_check_if_in_list "${VIDEO_NUM}" "${CLIP_SELECTION}")" == "y" ]] ; then
            # this clip is selected for the output
            VIDEO_NUM_EXT="${VIDEO_NUM}*"
            echo "[CHAPTER]" >> "${CHAPTER_LIST}"
            echo "TIMEBASE=1/100000" >> "${CHAPTER_LIST}"
            DURATION_S="$(_convert_hhmmssmmm2ns "${DURATION}")"
            END_VALUE=$((END_VALUE + DURATION_S))
            echo "START=${START_VALUE}" >> "${CHAPTER_LIST}"
            echo "END=${END_VALUE}" >> "${CHAPTER_LIST}"
            START_VALUE=$((START_VALUE + DURATION_S))
            if [[ -n "${MOD_TS}" && -n "${TIMECODE}" ]] ; then
                echo "title=${FILEBASENAME}: ${TIMECODE} - ${MOD_TS}" >> "${CHAPTER_LIST}"
            elif [[ -n "${MOD_TS}" ]] ; then
                echo "title=${FILEBASENAME}: ${MOD_TS}" >> "${CHAPTER_LIST}"
            elif [[ -n "${TIMECODE}" ]] ; then
                echo "title=${FILEBASENAME}: ${TIMECODE}" >> "${CHAPTER_LIST}"
            else
                echo "title=${FILEBASENAME}" >> "${CHAPTER_LIST}"
            fi
        else
            VIDEO_NUM_EXT="${VIDEO_NUM}"
        fi
        printf '%-7s %-7s %-27s %20s %16s %16s\n'  "${CARD_NUM}" "${VIDEO_NUM_EXT}" "${FILEBASENAME}" "${MOD_TS%.*}" "${DURATION}" "${TIMECODE}"
    done < <(tail -n +2 "${FILE_INVENTORY}" | grep ",video,")
    echo
}

_identify_card_type(){
    CARD="${1}"
    CARD_NAME="$(basename "${1}")"
    if [[ -n "$(find "${CARD}" -type d -iname "BDMV" | head -n 1)" ]] ; then
        _report -g "Camera card type identified: ${CARD_NAME} is AVCHD (e.g. Canon C100)"
        CAMERA_CARD_TYPE="AVCHD"
        CAMERA_CARD_DIR="$(find "${CAMERA_CARD_DIR[0]}" -type d -iname "BDMV" | head -n 1)" # set CAMERA_CARD_DIR path to the standard AVCHD directory structure
        EXTENSION="mts"
        RSYNC_OPTIONS=(-iname "*.cpi")
    elif [[ -n "$(find "${CARD}" -type d -iname "CLIPS001")" ]] ; then
        _report -g "Camera card type identified: ${CARD_NAME} is Canon XF (e.g. Canon C300)"
        CAMERA_CARD_TYPE="MXF"
        EXTENSION="mxf"
        RSYNC_OPTIONS=(\( -iname "*.xml" -o -iname "*.xmp" -o -iname "INDEX.MIF" -o -iname "*.cpf" \))
    elif [[ -n "$(find "${CARD}" -type f -iname "LASTCLIP.TXT")" ]] ; then
        _report -g "Camera card type identified: ${CARD_NAME} is P2 (Panasonic brand)"
        CAMERA_CARD_TYPE="P2"
        EXTENSION="mxf"
        RSYNC_OPTIONS=(\( -iname "*.xml" -o -iname "*.txt" -o -ipath "VOICE" \))
    elif [[ -n "$(find "${CARD}" \( -type d -iname "XDROOT" \) -o \( -type f -name "DISCMETA.XML" \) )" ]] ; then
        _report -g "Camera card type identified: ${CARD_NAME} is XAVC (Sony brand)"
        CAMERA_CARD_TYPE="XAVC"
        EXTENSION="mxf"
        # example (from an XDCam disc loaded with proav disabled):
        #├── ALIAS.XML
        #├── Clip
        #│   ├── C0001.MXF
        #│   ├── C0001M01.XML
        #│   ├── C0002.MXF
        #│   ├── C0002M01.XML
        #│   ├── C0003.MXF
        #│   └── C0003M01.XML
        #├── Component
        #├── DISCMETA.XML
        #├── Edit
        #├── General
        #│   └── Sony
        #│       └── Planning
        #├── INDEX.XML
        #├── MEDIAPRO.XML
        #├── SYSPRO.XML
        #├── Sub
        #│   ├── C0001S01.MXF
        #│   ├── C0002S01.MXF
        #│   └── C0003S0
        RSYNC_OPTIONS=(\( -iname "*.xml" -o -iname "*.bim" \))
    elif [[ -n "$(find "${CARD}" -type d -iname "BPAV")" ]] ; then
        _report -g "Camera card type identified: ${CARD_NAME} is XDCAM EX (e.g. Canon XF100)"
        CAMERA_CARD_TYPE="XDCAMEX"
        EXTENSION="mp4"
        RSYNC_OPTIONS=(\( -iname "*.xml" -o -iname "*.xmp" -o -iname "INDEX.MIF" -o -iname "*.cpf" \))
    else
        # read all the files in the package and note the extensions of video files
        EXTENSION_REVIEW=()
        while read FILE ; do
            if [[ -n $(mediainfo --Inform="General;%VideoCount%" "${FILE}") ]] ; then
                EXTENSION_REVIEW+=("${FILE##*.}")
            fi
        done < <(find "${CARD}" -type f | sort)
        # find the most common video extension
        EXTENSION="$(printf '%s\n' "${EXTENSION_REVIEW[@]}" | sort | uniq -c | sort -k1,1nr -k2 | awk '{print $2; exit}')"
        EXTENSION_COUNT="$(printf '%s\n' "${EXTENSION_REVIEW[@]}" | sort | uniq -c | sort -k1,1nr -k2 | awk '{print $1; exit}')"
        _report -g "Couldn't identify a particular camera card type here. Using a GENERAL approach on the ${EXTENSION_COUNT} ${EXTENSION} files found in there."
        _report -g "If you want to keep everything in ${CARD} in a tar.gz file then try the -t optoin."
        CAMERA_CARD_TYPE="GENERAL"
    fi
}

_check_if_in_list(){
    VALUE="${1}"
    LIST="${2}" # list is expected as comma-delimited
    unset LIST_EXPANDED
    # inspired by https://unix.stackexchange.com/posts/463617/revisions
    IFS=',' read -a RANGES  <<< "${LIST}"
    for RANGE in "${RANGES[@]}"; do
        if [[ ${RANGE} == *"-"* ]] ; then
            IFS=- read START END <<< "${RANGE}"
            [ -z "${START}" ] && continue
            [ -z "${END}" ] && END="${START}"
            for (( i=START ; i <= END ; i++ )); do
                LIST_EXPANDED+=",${i}"
            done
        else
            LIST_EXPANDED+=",${RANGE}"
        fi
    done
    if [[ ",${LIST_EXPANDED}," = *",${VALUE},"* ]] ; then
        echo "y"
    else
        echo "n"
    fi
}

# getopts loop
OPTIND=1
SUFFIX="_concatenated"
while getopts ":m:N:o:f:F:ctans:k:p:1h" OPT; do
    case "${OPT}" in
        m) MEDIAID="${OPTARG}" ;;
        N) CLIP_SELECTION="${OPTARG}" ;;
        o) AIP_DESTINATION="${OPTARG}" ;;
        f) OUTPUT_EXTENSION="${OPTARG}" ;;
        F) FFMPEG_PATH="${OPTARG}" ;;
        c) _report "The -c option (which combined 2 cards) is now removed. Please simply list as many camera cards as inputs as you need." ; exit ;;
        t) TAR="YES" ; AIP="NO" ;;
        a) TAR="YES" ; AIP="YES" ;;
        n) REPORT_ONLY="Y" ;;
        s) SUFFIX="${OPTARG}" ;;
        k) KEEP_ALL="${OPTARG}" ;;
        p) PROFILE="${OPTARG}" ;;
        1) SINGLE_OUTPUT="Y" ;;
        h) _usage ;;  # if the operator runs "[scriptname] -h" then the _usage text above will display in the terminal
        *) echo "Invalid option -${OPTARG}" ; _usage ;; # if the operator tries to use an option other than the ones listed above, the _usage text will display in the terminal
    esac
done
shift $(( OPTIND - 1 ))

trap _cleanup SIGHUP SIGINT SIGTERM # "termination signals" that ask the system to clean up and safely kill the ongoing process, with increasing urgency

# check for unassigned variables
CAMERA_CARD_DIR=("${@}")
# ask for camera card if it wasn't supplied
if [[ -z "${CAMERA_CARD_DIR[0]}" ]] ; then
    _report -b "Drag in the input directory or type 'q' to quit: "
    read -e CAMERA_CARD_DIR
    [[ "${CAMERA_CARD_DIR}" = "q" ]] && exit 0
fi
# check that each camera card is a directory
for CARD_DIR in "${CAMERA_CARD_DIR[@]}" ; do
    if [[ ! -d "${CARD_DIR}" ]] ; then
        _report -rt "ERROR: Input directory ${CAMERA_CARD_DIR} is not a directory. Exiting..."
        exit 1
    fi
done

# create temp filelists for use during script
FILE_INVENTORY="$(_maketemp).csv"
TEMP_CONCATLIST="$(_maketemp).ffconcat"

_make_inventory
_print_table
if [[ "${REPORT_ONLY}" = "Y" ]] ; then
    exit 0
fi

# determine FFMPEG_PATH
if [[ -z "${FFMPEG_PATH}" ]] ; then
    FFMPEG_PATH="$(which ffmpeg)"
fi
# ask for MEDIAID if it wasn't supplied
if [[ -z "${MEDIAID}" ]] ; then
    _report -b -n "Enter a unique package name or 'q' to quit: "
    read -e MEDIAID
    [[ "${MEDIAID}" = "q" ]] && exit 0
fi
# ask for AIP_DESTINATION if it wasn't supplied
if [[ -z "${AIP_DESTINATION}" ]] ; then
    _report -b -n "Provide the destination for the resulting package or 'q' to quit: "
    read -e AIP_DESTINATION
    echo
    [[ "${AIP_DESTINATION}" = "q" ]] && exit 0
    # check that camera card is a directory
    if [[ ! -d "${AIP_DESTINATION}" ]] ; then
        _report -rt "ERROR: Output directory ${AIP_DESTINATION} is not a directory. Exiting..."
        exit 1
    fi
fi

# note the time we start working on the AIP
START_TIME="$(date +%FT%T)"

# define output directories
if [[ "${SINGLE_OUTPUT}" = "Y" ]] ; then
    AIPDIR="${AIP_DESTINATION}"
else
    AIPDIR="${AIP_DESTINATION}/${MEDIAID}"
fi
if [[ "${SINGLE_OUTPUT}" = "Y" ]] ; then
    AIPDIR_AIP="${AIPDIR}"
    METADATADIR="${AIPDIR}"
    LOG="${METADATADIR}/${MEDIAID}_$(basename "${0}")_$(date +%F-%H%M%S).log"
    METADATA_REPORTS_DIR="${METADATADIR}"
    METADATA_ORIGINAL_CAMERA_FILES_DIR="${METADATADIR}"
    mkdir -p "${AIPDIR_AIP}"
else
    METADATADIR="${AIPDIR}/metadata"
    LOG="${METADATADIR}/$(basename "${0}")_$(date +%F-%H%M%S).log"
    METADATA_REPORTS_DIR="${METADATADIR}/reports"
    METADATA_ORIGINAL_CAMERA_FILES_DIR="${METADATADIR}/original_camera_files"
    if [[ -d "${AIPDIR}" ]] ; then
        _report -rt "ERROR: Output directory ${AIPDIR} already exists! Exiting to avoid overwriting..."
        exit 1
    fi
    if [[ "${AIP}" == "YES" && "${TAR}" == "YES" ]] ; then
        AIPDIR_AIP="${AIPDIR}/objects/AIP"
        AIPDIR_TAR="${AIPDIR}/objects/TAR"
        mkdir -p "${METADATA_REPORTS_DIR}" "${METADATA_ORIGINAL_CAMERA_FILES_DIR}" "${AIPDIR_AIP}" "${AIPDIR_TAR}"
    elif [[ "${AIP}" == "YES" ]] ; then
        AIPDIR_AIP="${AIPDIR}/objects"
        mkdir -p "${METADATA_REPORTS_DIR}" "${METADATA_ORIGINAL_CAMERA_FILES_DIR}" "${AIPDIR_AIP}"
    elif [[ "${TAR}" == "YES" ]] ; then
        AIPDIR_TAR="${AIPDIR}/objects/TAR"
        mkdir -p "${METADATA_REPORTS_DIR}" "${AIPDIR_TAR}"
    else
        _report -r "ERROR: camera_cards needs to produce either an AIP or TAR output."
        exit 1
    fi
fi
### procedure for creating structured AIPs (concatenating video files and restructuring metadata directories)
if [[ "${AIP}" == "YES" ]] ; then
    if [[ "${SINGLE_OUTPUT}" != "Y" ]] ; then
        mkdir -p "${METADATA_ORIGINAL_CAMERA_FILES_DIR}"
        # copy selected original metadata files to AIP
        _report -g "Moving significant camera-generated metadata files to AIP..."
        CAMERA_CARD_METADATA2KEEP=()
        while read METADATA_FILE ; do
            CAMERA_CARD_METADATA2KEEP+=("${METADATA_FILE}")
        done < <(_read_inventory -r "${CLIP_SELECTION}" -f ",metadata," "FILE_NAME")
        if [[ -w "${METADATA_ORIGINAL_CAMERA_FILES_DIR}" ]] ; then
            if (( ${#CAMERA_CARD_METADATA2KEEP[@]} != 0 )) ; then
                _report -g "Copying some camera metadata files into the package."
                rsync -avh "${CAMERA_CARD_METADATA2KEEP[@]}" "${METADATA_ORIGINAL_CAMERA_FILES_DIR}"
            fi
        else
            _report -r "Error: the folder for camera metadata is not writable."
            exit 1
        fi

        # generate metadata reports
        _generate_metadata
    fi

    ## concatenate video files into a single file
    echo
    FIRST_FILE="$(_read_inventory -f ",video," "FILE_NAME" | head -n 1)"
    if [[ -z "${OUTPUT_EXTENSION}" ]] ; then
        if [[ "${CAMERA_CARD_TYPE}" == "GENERAL" ]] ; then
            OUTPUT_EXTENSION="${FIRST_FILE#*.}"
        else
            OUTPUT_EXTENSION="${EXTENSION}"
        fi
    fi
    _report -g "Concatenating video files into a ${OUTPUT_EXTENSION} file."
    CONCATENATED_VIDEO_FILE="${AIPDIR_AIP}/${MEDIAID}${SUFFIX}.${OUTPUT_EXTENSION}"

    # create list of files to concatenate, formatted for ffmpeg
    unset AUDIO_STRATEGY UNKNOWN_ENCODINGS_FOUND
    while read INVENTORY_ROW ; do
        SEG_NUMBER="$(echo "${INVENTORY_ROW}" | cut -d "," -f 3)"
        FILE_NAME="$(echo "${INVENTORY_ROW}" | cut -d "," -f 12)"
        AUDIO_CODEC="$(echo "${INVENTORY_ROW}" | cut -d "," -f 13)"
        AUDIO_SAMPLE_FMT="$(echo "${INVENTORY_ROW}" | cut -d "," -f 14)"
        if [[ "${AUDIO_CODEC}" == "pcm_bluray" && "${OUTPUT_EXTENSION}" != "m2t" ]] ; then
            if [[ "${AUDIO_SAMPLE_FMT}" == "s16" ]] ; then
                _report -b "Note: A file contains ${AUDIO_CODEC}(${AUDIO_SAMPLE_FMT}) audio but the output is output is not m2t, we'll transcode to pcm_s16le."
                AUDIO_STRATEGY=(-c:a pcm_s16le)
            else
                _report -b "Note: A file contains ${AUDIO_CODEC}(${AUDIO_SAMPLE_FMT}) audio but the output is output is not m2t, we'll transcode to pcm_s24le."
                AUDIO_STRATEGY=(-c:a pcm_s24le)
            fi
        fi
        if [[ -z "${CLIP_SELECTION}" || "$(_check_if_in_list "${SEG_NUMBER}" "${CLIP_SELECTION}")" == "y" ]] ; then
            ANY_UNKNOWN_ENCODINGS="$(ffprobe -v 0 "${FILE_NAME}" -show_streams | grep -c codec_name=unknown)"
            if [[ "${ANY_UNKNOWN_ENCODINGS}" != "0" && "${UNKNOWN_ENCODINGS_FOUND}" != true ]] ; then
                UNKNOWN_ENCODINGS_FOUND=true
                MIDDLE_OPTIONS_ALL=(-ignore_unknown)
                _report -b "Uknown encodings were found in ${FILE_NAME}. Will add -ignore_unknown while rewrapping."
            fi
            echo "file '${FILE_NAME}'" >> "${TEMP_CONCATLIST}"
        fi
    done < <(_read_inventory -f ",video,\|,audio,")

    # concatenate video files in the order they are printed in $TEMP_CONCATLIST; map metadata from the first video file (in sequence) onto the concatenated file
    if [[ "${PROFILE}" = "xn" ]] ; then
        MIDDLE_OPTIONS+=(-r:v ntsc -c:v mpeg2video -intra_vlc 1 -non_linear_quant 1)
        MIDDLE_OPTIONS+=(-pix_fmt yuv420p -flags +ildct+ilme -top 1 -dc 10 -qmin 1 -lmin QP2LAMBDA -b:v 35000k -minrate 20000k -maxrate 35000k -bufsize 36408333 -g 15 -bf 2 -aspect 16:9)
        MIDDLE_OPTIONS+=(-ps 1 -mpv_flags strict_gop)
        MIDDLE_OPTIONS+=(-s 1920x1080)
        MIDDLE_OPTIONS+=(-acodec pcm_s16le -ar 48000)
    elif [[ "${PROFILE}" = "xp" ]] ; then
        MIDDLE_OPTIONS+=(-r:v pal -c:v mpeg2video -intra_vlc 1 -non_linear_quant 1)
        MIDDLE_OPTIONS+=(-pix_fmt yuv420p -flags +ildct+ilme -top 1 -dc 10 -qmin 1 -lmin QP2LAMBDA -b:v 35000k -minrate 20000k -maxrate 35000k -bufsize 36408333 -g 15 -bf 2 -aspect 16:9)
        MIDDLE_OPTIONS+=(-ps 1 -mpv_flags strict_gop)
        MIDDLE_OPTIONS+=(-s 1920x1080)
        MIDDLE_OPTIONS+=(-acodec pcm_s16le -ar 48000)
    elif [[ "${PROFILE}" = "h" ]] ; then
        MIDDLE_OPTIONS+=(-movflags faststart -pix_fmt yuv420p -c:v libx264 -force_key_frames chapters -crf 18 -vf yadif -c:a aac)
    else
        MIDDLE_OPTIONS+=(-c copy)
    fi
        
    "${FFMPEG_PATH}" -f concat -safe 0 -i "${TEMP_CONCATLIST}" -i "${FIRST_FILE}" -map 0 -map_metadata 1 "${MIDDLE_OPTIONS_ALL[@]}" "${MIDDLE_OPTIONS[@]}" "${AUDIO_STRATEGY[@]}" "${CONCATENATED_VIDEO_FILE}"
    # calculate md5 for each stream in the file
    if [[ "${SINGLE_OUTPUT}" != "Y" ]] ; then
        _report -b "Calculating streamhash and checking input for errors."
        DECODER_LOG="${METADATA_REPORTS_DIR}/$(basename "${CONCATENATED_VIDEO_FILE%.*}")_decoder_log.txt"
        STREAMHASH="${METADATA_REPORTS_DIR}/$(basename "${CONCATENATED_VIDEO_FILE%.*}")_streamhash.md5"
        "${FFMPEG_PATH}" -debug_ts -f concat -safe 0 -i "${TEMP_CONCATLIST}" -map 0 -c copy -f streamhash -hash md5 "${MIDDLE_OPTIONS_ALL[@]}" "${STREAMHASH}" 2> "${DECODER_LOG}"

        # decode error summary
        FIRST_VIDEO_FILE="$(_read_inventory -f ",video," "FILE_NAME" | head -n 1)"
        VIDEO_FORMAT="$(ffprobe -v 0 -i "${FIRST_VIDEO_FILE}" -show_entries format=format_name -of default=nw=1:nk=1)"
        VIDEO_ENCODING="$(ffprobe -v 0 -i "${FIRST_VIDEO_FILE}" -select_streams v:0 -show_entries stream=codec_name -of default=nw=1:nk=1 | head -n 1)"
        DEMUXER_ERROR_SUMMARY="$(grep -B 1 "^\[${VIDEO_FORMAT} @" "${DECODER_LOG}" | grep "^\[${VIDEO_FORMAT} @\|pts_time")"
        DECODER_ERROR_SUMMARY="$(grep -B 1 "^\[${VIDEO_ENCODING} @" "${DECODER_LOG}" | grep "^\[${VIDEO_ENCODING} @\|pts_time")"
        if [[ -n "${DECODER_ERROR_SUMMARY}" ]] ; then
            DECODER_LOG_SUMMARY="${METADATA_REPORTS_DIR}/$(basename "${CONCATENATED_VIDEO_FILE%.*}")_decoder_log_summary.txt"
            _report -r "Warning there's some decoding errors. If a timestamp is referenced here please review that portion of the output."
            echo "${DECODER_ERROR_SUMMARY}"
            echo "${DECODER_ERROR_SUMMARY}" > "${DECODER_LOG_SUMMARY}"
        fi
        if [[ -n "${DEMUXER_ERROR_SUMMARY}" ]] ; then
            DEMUXER_LOG_SUMMARY="${METADATA_REPORTS_DIR}/$(basename "${CONCATENATED_VIDEO_FILE%.*}")_demuxer_log_summary.txt"
            _report -r "Warning there's some demuxer errors. If a timestamp is referenced here please review that portion of the output."
            echo "${DEMUXER_ERROR_SUMMARY}"
            echo "${DEMUXER_ERROR_SUMMARY}" > "${DEMUXER_LOG_SUMMARY}"
        fi

        # tests for video file existing
        FFMPEG_EXIT_CODE="${?}"
        if [[ ! -s "${CONCATENATED_VIDEO_FILE}" ]] ; then
            ((FFMPEG_ERRORS++))
        fi
        _write_package_log "CONCATENATED VIDEO FILE" "${CONCATENATED_VIDEO_FILE}"
        _report -g "Video concatenation process finished"
    fi

    # if there are audio-only files, concatenate them and join with concatenated video file
    if [[ -n "$(_read_inventory -f ",audio," "FILE_NAME")" ]] ; then
        _report -g "Concatenating audio files..."
        FIRST_FILE="$(_read_inventory -f ",audio," "FILE_NAME" | head -n 1)"
        CONCATENATED_AUDIO_FILE="${AIPDIR_AIP}/${MEDIAID}${SUFFIX}_audio.${FIRST_FILE#*.}"
        > "${TEMP_CONCATLIST}" # clear concatlist from video concatenation
        while read FILE ; do
            echo "file '${FILE}'" >> "${TEMP_CONCATLIST}"
        done < <(_read_inventory -f ",audio," "FILE_NAME")
        "${FFMPEG_PATH}" -f concat -safe 0 -i "${TEMP_CONCATLIST}" -i "${FIRST_FILE}" -map 0 -map_metadata 1 -c copy "${MIDDLE_OPTIONS_ALL[@]}" "${CONCATENATED_AUDIO_FILE}"
        _write_package_log "CONCATENATED AUDIO FILE" "${CONCATENATED_AUDIO_FILE}"

        # merge video and audio
        _report -g "Merging concatenated audio and video files..."
        # check that video and audio file durations match
        VIDEO_DURATION=$(mediainfo --Inform="Video;%Duration%" "${CONCATENATED_VIDEO_FILE}")
        AUDIO_DURATION=$(mediainfo --Inform="Audio;%Duration%" "${CONCATENATED_AUDIO_FILE}")
        if [[ "${VIDEO_DURATION}" -ne "${AUDIO_DURATION}" ]] ; then
            _report -r "Audio and video files are not the same length! Review output for sync. (Video duration = ${VIDEO_DURATION}, audio duration = ${AUDIO_DURATION})"
        fi
        # rename concatenated video file to avoid collision with audio file
        RENAMED_CONCATENATED_VIDEO_FILE="${CONCATENATED_VIDEO_FILE//concatenated/concatenated_video}"
        mv "${CONCATENATED_VIDEO_FILE}" "${RENAMED_CONCATENATED_VIDEO_FILE}"
        CONCATENATED_FILE_MERGED="${AIPDIR_AIP}/${MEDIAID}${SUFFIX}.${FIRST_FILE#*.}"
        "${FFMPEG_PATH}" -i "${RENAMED_CONCATENATED_VIDEO_FILE}" -i "${CONCATENATED_AUDIO_FILE}" -c:v copy -c:a copy "${MIDDLE_OPTIONS_ALL[@]}" "${CONCATENATED_FILE_MERGED}"
        _write_package_log "JOINED FILE (AUDIO AND VIDEO)" "${CONCATENATED_FILE_MERGED}"
        _report -g "Audio + video joining process ended"
    fi

    if [[ "${SINGLE_OUTPUT}" != "Y" ]] ; then
        # generate md5 of concatenated file
        if [[ -f "${CONCATENATED_FILE_MERGED}" ]] ; then
            CONCATENATED_MD5_OUTPUT="${METADATA_REPORTS_DIR}/$(basename "${CONCATENATED_FILE_MERGED%.*}").md5"
            md5sum "${CONCATENATED_FILE_MERGED}" > "${CONCATENATED_MD5_OUTPUT}"
        else
            CONCATENATED_MD5_OUTPUT="${METADATA_REPORTS_DIR}/$(basename "${CONCATENATED_VIDEO_FILE%.*}").md5"
            md5sum "${CONCATENATED_VIDEO_FILE}" > "${CONCATENATED_MD5_OUTPUT}"
        fi

        if [[ "${CAMERA_CARD_TYPE}" == "GENERAL" ]] ; then
            # general profile only: move all video files to AIP as separate files, in case concatenation does not work for this camera card structure
            DUPLICATE_NAMES="$(while read FILE ; do basename "${FILE}" ; done < <(_read_inventory "FILE_NAME") | sort | uniq -d)"
            if [[ -n "${DUPLICATE_NAMES}" ]] ; then # check for duplicate filenames on camera card before beginning transfer process
                _report -r "Duplicate filenames found on original camera card! Not completing automated packaging process in order to avoid overwriting files with the same name. Please take a closer look at your original camera card's file structure to create this AIP."
                _report -r "Also note that metadata files (in the nmaahc_metadata folder of your AIP) may not have been properly generated for all files with the same filename."
                exit 1
            else
                if [[ "${KEEP_ALL}" = "Y" ]] ; then
                    echo
                    _report -g "Moving original audiovisual files to AIP..."
                    while read FILE ; do
                        rsync -avh --progress "${FILE}" "${AIPDIR_AIP}/"
                    done < <(_read_inventory -f ",video,\|,audio," "FILE_NAME")
                fi
            fi
        fi
    fi
fi

### procedure for packaging files into compressed tars
if [[ "${TAR}" == "YES" ]] ; then
    ## create metadata reports for original package and audiovisual files
    # generate metadata reports
    _generate_metadata

    ## tar supplied input directory
    echo
    _report -g "Creating tar archive..."
    tar -czvf "${AIPDIR_TAR}/${MEDIAID}.tar.gz" -C "${CAMERA_CARD_DIR}" .
    _write_package_log "TAR ARCHIVE" "${MEDIAID}.tar.gz"

    ## generate checksum for tar archive
    TAR_MD5_OUTPUT="${METADATA_REPORTS_DIR}/${MEDIAID}.md5"
    md5sum "${AIPDIR_TAR}/${MEDIAID}.tar.gz" > "${TAR_MD5_OUTPUT}"
fi

# record variables in ingest log
_write_package_log "OPERATOR" "${OP}"
_write_package_log "CAMERA CARD DIRECTORY" "${CAMERA_CARD_DIR[@]}"
_write_package_log "MEDIAID" "${MEDIAID}"
_write_package_log "OUTPUT DIRECTORY" "${AIPDIR}"
_write_package_log "CAMERA CARD TYPE" "${CAMERA_CARD_TYPE}"
_write_package_log "STRATEGY" "${STRATEGY}"
_write_package_log "START TIME" "${START_TIME}"

# error reporting
echo
if [[ "${SINGLE_OUTPUT}" != "Y" ]] ; then
    if [[ ! -s "${TREE}" ]] ; then
        _report -r "Metadata generation process created one or more blank files. ${TREE} may be empty. Review metadata output at ${METADATA_REPORTS_DIR}."
        _write_package_log "POSSIBLE_ERROR_REVIEW" "Metadata generation process created one or more blank files"
    fi
    if [[ "${FFMPEG_EXIT_CODE}" -gt 0 ]] || [[ "${FFMPEG_ERRORS}" -gt 0 ]] ; then
        _report -r "FFmpeg may have encountered errors - review terminal output for more detail."
    else
        _report -g "FFmpeg exit codes look ok"
        if [[ -s "${CONCATENATED_AUDIO_FILE}" ]] ; then
            if [[ "${VIDEO_DURATION}" -ne "${AUDIO_DURATION}" ]] ; then
            _report -r "Concatenated audio and video files were not the same length - review final merged file for sync at ${CONCATENATED_FILE_MERGED} (video duration = ${VIDEO_DURATION}, audio duration = ${AUDIO_DURATION})"
            _write_package_log "POSSIBLE_ERROR_REVIEW" "Concatenated audio and video files were not the same length"
            else
                _report -g "Concatenated audio and video process looks ok"
            fi
        fi
        if [[ "${AIP}" == "YES" ]] ; then # check for losslessness of concatenated file by comparing bitstream md5s
            OUTPUT_STREAMHASH="$(_maketemp)"
            _report -b "Checking streamhash between input and output..."
            "${FFMPEG_PATH}" -y -loglevel quiet -report -i "${CONCATENATED_VIDEO_FILE}" -map 0 -c copy -f streamhash -hash md5 "${MIDDLE_OPTIONS_ALL[@]}" "${OUTPUT_STREAMHASH}"
            if [[ $(cat "${STREAMHASH}") != $(cat "${OUTPUT_STREAMHASH}") ]] ; then
                _report -r "Concatenation process was not lossless! Review final merged file at ${CONCATENATED_VIDEO_FILE}"
                _write_package_log "POSSIBLE_ERROR_REVIEW" "Concatenation process was not lossless based on bitstream md5s"
            else
                _report -g "Concatenated process looks lossless"
            fi
        fi
    fi
fi

# log script ending
_report -g "camera_cards process complete. Your package can be found at ${AIPDIR}. A log file can be found at ${LOG}"