ingestfile

#!/usr/bin/env bash

# a script to turn SIPs into AIPs

# load nmaahcmmfunctions into this script
SCRIPT_PATH="${0%/*}"
. "${SCRIPT_PATH}/nmaahcmmfunctions"
[[ -f "${SCRIPT_PATH}/nmaahcmmfunctions" ]] || { echo "Missing '${SCRIPT_PATH}/nmaahcmmfunctions'. Exiting." ; exit 1 ;};
_initialize_make # safe script termination process defined in nmaahcmmfunctions
RSYNC_ERROR_COUNT=0
LOG="${LOGDIR%/}/$(basename "${0}")_$(date +%F-%H%M%S).log"

## USAGE

_usage(){
    echo
    echo "$(basename "${0}")"
    echo "This script will run an interactive interview and turn an input package (SIP) into an output package (AIP) accordingly."
    echo "Your output package (AIP) will be delivered to the AIP destination set in nmaahcmmconfig, which is currently ${AIP_DESTINATION}"
    echo "Dependencies: ${DEPENDENCIES[@]}"
    echo "Usage: $(basename ${0}) [ -options ]"
    echo "  Local mode: presents a graphical user interface to specify file, media ID, and other options. Automatically creates an H264 access copy and generates metadata files if not already present."
    echo "  -h display this help"
    echo
    exit
}
# getopts loop
OPTIND=1
while getopts ":h" OPT; do
    case "${OPT}" in
        h) _usage ;;  # if the operator runs "[scriptname] -h" then the _usage text above will display in the terminal
        *) echo "Invalid option -${OPTARG}" ; _usage ;; # if the operator tries to use an option other than the ones listed above, the _usage text will display in the terminal
    esac
done
shift $(( ${OPTIND} - 1 ))

## PASHUA INTERFACE

CONF="
# Set transparency: 0 is transparent, 1 is opaque
*.transparency=1.00

# Set window title
*.title = Ingest File

# intro text
intro.x = 20
intro.y = 480
intro.width = 500
intro.type = text
intro.text = Ingest file options. Leave the option blank to be prompted.

# ask operator
OP.x = 20
OP.y = 420
OP.type = textfield
OP.label = Please enter your name:
OP.width = 250

# input directory
INPUT.x = 20
INPUT.y = 360
INPUT.type = openbrowser
INPUT.label = Select your input package (SIP):
INPUT.width=400

# package type
PACKAGETYPE.x = 20
PACKAGETYPE.y = 300
PACKAGETYPE.type = combobox
PACKAGETYPE.label = Please indicate the type of package being submitted [scroll for all options].
PACKAGETYPE.option =
PACKAGETYPE.option = Digitized Film (DPX package)
PACKAGETYPE.option = Digitized Film (MOV, MKV master files)
PACKAGETYPE.option = Digitized Analog Video (vrecord package)
PACKAGETYPE.option = Transferred DV (MOV, MKV master files)
PACKAGETYPE.option = Great Migration (various formats)
PACKAGETYPE.option = Other/Unknown
PACKAGETYPE.width = 400

# mediaid
MEDIAID.x = 20
MEDIAID.y = 240
MEDIAID.type = textfield
MEDIAID.label = Enter a unique media ID/package name ([aA-zZ][0-9]-_ only).
MEDIAID.width = 250

# destination
DESTINATION.x = 20
DESTINATION.y = 180
DESTINATION.type = openbrowser
DESTINATION.label = Select where the package will go (AIP destination).
DESTINATION.width = 400

# verifySIP
VERIFYSIP.x = 20
VERIFYSIP.y = 120
VERIFYSIP.type = combobox
VERIFYSIP.label = Select whether to run verifySIP prior to ingest, to check for package conformance.
VERIFYSIP.default = Run verifySIP
VERIFYSIP.option =
VERIFYSIP.option = Run verifySIP
VERIFYSIP.option = Don't run verifySIP
VERIFYSIP.width = 400

# cleanup decision
CLEANUPDECISION.x = 20
CLEANUPDECISION.y = 60
CLEANUPDECISION.type = combobox
CLEANUPDECISION.label = Select a cleanup strategy.
CLEANUPDECISION.default = Leave source files where they are
CLEANUPDECISION.option =
CLEANUPDECISION.option = Leave source files where they are
CLEANUPDECISION.option = Remove source files after successful ingest
CLEANUPDECISION.width = 400

# Add a cancel button with default label
cb.type=cancelbutton

";

# run Pashua
PASHUA_CONFIGFILE=`/usr/bin/mktemp /tmp/pashua_XXXXXXXXX`
echo "${CONF}" > "${PASHUA_CONFIGFILE}"
_pashua_run
rm "${PASHUA_CONFIGFILE}"
echo "  OPERATOR = ${OP}"
echo "  INPUT = ${INPUT}"
echo "  MEDIAID = ${MEDIAID}"
echo "  PACKAGETYPE = ${PACKAGETYPE}"
echo "  DESTINATION = ${DESTINATION}"
echo "  VERIFYSIP = ${VERIFYSIP}"
echo "  CLEANUPDECISION = ${CLEANUPDECISION}"
echo ""

# check that all inputs were entered
if [[ -z "${OP}" ]] ; then
    printf "Enter the name of the operator or 'q' to quit: "
    read -e OP
    [[ "${OP}" = "q" ]] && exit 0
fi
if [[ -z "${INPUT}" ]] ; then
    printf "Drag in the input directory or type 'q' to quit: "
    read -e INPUT
    [[ "${INPUT}" = "q" ]] && exit 0
    if [[ ! -d "${INPUT}" ]] ; then
        _report -rt "ERROR: Input ${INPUT} is not a directory. Exiting..."
        _log -a "Process terminated by script (input was not a directory)."
        exit 1
    fi
    basename=$(basename "${INPUT}")
fi
if [[ -z "${PACKAGETYPE}" ]] ; then
    printf "Select a package type:"
    echo
    PS3="Selection: "
    select PACKAGETYPE in "Digitized Film (DPX package)" "Digitized Film (MOV, MKV master files)" "Digitized Analog Video (vrecord package)" "Transferred DV (MOV, MKV master files)" "Great Migration (various formats)" "Other/Unknown" "quit"
    do
        if [[ "${PACKAGETYPE}" == "quit" ]] ; then echo "Bye" && exit 0 ; fi
        break
    done
fi
if [[ -z "${MEDIAID}" ]] ; then
    printf "Enter a unique package name or 'q' to quit: "
    read -e MEDIAID
    [[ "${MEDIAID}" = "q" ]] && exit 0
    # validate id and perhaps fail with exit
    [[ -z "${MEDIAID}" ]] && { _report -rt  "ERROR: You must enter a valid MEDIA ID" ; exit ;};
    [[ ! -z $(echo "${MEDIAID}" | grep -v "^[A-Za-z0-9_-]*$") ]] && { _report -rt  "ERROR The MEDIA ID must only contain letters, numbers, hyphen and underscore" ; exit 1 ;};
fi
if [[ -z "${DESTINATION}" ]] ; then
    printf "Drag in the destination directory or type 'q' to quit: "
    read -e DESTINATION
    [[ "${DESTINATION}" = "q" ]] && exit 0
    if [[ ! -d "${DESTINATION}" ]] ; then
        _report -rt "ERROR: Destination ${DESTINATION} is not a directory. Exiting..."
        _log -a "Process terminated by script (input was not a directory)."
        exit 1
    fi
fi
if [[ -z "${VERIFYSIP}" ]] ; then
    printf "Select whether to run verifySIP prior to ingest, to check for package conformance:"
    echo
    select VERIFYSIP in "Run verifySIP" "Don't run verifySIP" "quit"
    do
        if [[ "${VERIFYSIP}" == "quit" ]] ; then echo "Bye" && exit 0 ; fi
        break
    done
fi
if [[ -z "${CLEANUPDECISION}" ]] ; then
    printf "Select a cleanup strategy:"
    echo
    select CLEANUPDECISION in "Leave source files where they are" "Remove source files after successful ingest" "quit"
    do
        if [[ "${CLEANUPDECISION}" == "quit" ]] ; then echo "Bye" && exit 0 ; fi
        break
    done
fi

## SCRIPT ACTIONS

# log script beginning
_log -b

# record variables in ingest log
_writelog "OPERATOR" "${OP}"
_writelog "INPUT" "${INPUT}"
_writelog "MEDIAID" "${MEDIAID}"
_writelog "PACKAGETYPE" "${PACKAGETYPE}"
_writelog "DESTINATION" "${DESTINATION}"
_writelog "VERIFYSIP" "${VERIFYSIP}"
_writelog "CLEANUPDECISION" "${CLEANUPDECISION}"
_writelog "START TIME" "$(date +%FT%T)"
_writelog " " " "

# make AIP directory
mkdir -p "${DESTINATION}/${MEDIAID}"
# remove .DS_Store files and log action if successful
"${SCRIPTDIR}/removeDSStore" "${INPUT}" && _writelog ".DS_Store files removed by invoking removeDSStore" "$(date +%FT%T)"

if [[ "${VERIFYSIP}" == "Run verifySIP" ]] ; then
    _report -g "Running verifySIP..."
    if [[ "${PACKAGETYPE}" == "Digitized Film (DPX package)" ]] ; then
        "${SCRIPT_PATH}/verifySIP" -x "${INPUT}"
    elif [[ "${PACKAGETYPE}" == "Digitized Film (MOV, MKV master files)" ]] ; then
        "${SCRIPT_PATH}/verifySIP" -f "${INPUT}"
    elif [[ "${PACKAGETYPE}" == "Digitized Analog Video (vrecord package)" ]] ; then
        "${SCRIPT_PATH}/verifySIP" -v "${INPUT}"
    elif [[ "${PACKAGETYPE}" == "Transferred DV (MOV, MKV master files)" ]] ; then
        "${SCRIPT_PATH}/verifySIP" -d "${INPUT}"
    elif [[ "${PACKAGETYPE}" == "Great Migration (various formats)" ]] ; then
        _report -r "Currently verifySIP does not support Great Migration packages."
    elif [[ "${PACKAGETYPE}" == "Other/Unknown" ]] ; then
        "${SCRIPT_PATH}/verifySIP" -u "${INPUT}"
    fi
    _report -r "Based on the output of verifySIP, do you want to continue with the ingestfile process?"
    printf "Select an option:"
    echo
    PS3="Selection: "
    select ACTION in "Continue" "Cancel - don't do anything to my package and end ingestfile process"
    do
        break
    done
    if [[ "${ACTION}" == "Continue" ]] ; then
        :
    elif [[ "${ACTION}" == "Cancel - don't do anything to my package and end ingestfile process" ]] ; then
        exit
    fi
else
    :
fi

if [[ "${PACKAGETYPE}" == "Digitized Film (DPX package)" ]] ; then
    # if working with a DPX package, check to see if it's a deprecated package structure; new packages should have directories called MEDIAID_DPX, MEDIAID_Audio, and MEDIAID_Derivatives
    if [[ -n $(find "${INPUT}" -iname "*MOV" -type d) ]] | [[ -n $(find "${INPUT}" -iname "*WAV" -type d) ]] ; then
        _report -r "Your DPX directory structure may be outdated! Do you want to run restructureDPX?"
        _report -r "If you run restructureDPX, your DPX will be repackaged according to the updated structure and moved to the destination you selected, after which the ingestfile process will continue."
        printf "Select an option:"
        echo
        PS3="Selection: "
        select ACTION in "Run restructureDPX" "Continue without running restructureDPX" "Cancel - don't do anything to my package and end ingestfile process"
        do
            break
        done
        if [[ "${ACTION}" == "Run restructureDPX" ]] ; then
            # run restructureDPX microservice
            if [[ ! -d "${DESTINATION}/${MEDIAID}/objects" ]] ; then mkdir -p "${DESTINATION}/${MEDIAID}/objects" ; fi
            echo
            _report -sb "Invoking restructureDPX..."
            if [[ "${CLEANUPDECISION}" == "Remove source files after successful ingest" ]] ; then
                "${SCRIPT_PATH}/restructureDPX" -d "${DESTINATION}/${MEDIAID}/objects" -c "${INPUT}"
            else
                "${SCRIPT_PATH}/restructureDPX" -d "${DESTINATION}/${MEDIAID}/objects" "${INPUT}"
            fi
            _writelog "Invoked restructureDPX:" "$(date +%FT%T)"
        elif [[ "${ACTION}" == "Continue without running restructureDPX" ]] ; then
            :
        elif [[ "${ACTION}" == "Cancel - don't do anything to my package and end ingestfile process" ]] ; then
            exit
        fi
    else
        _report -g "Moving DPX package as-is to specified destination..."
        if [[ "$(basename ${INPUT})" == "${MEDIAID}" ]] ; then
            rsync -avch --progress "${INPUT}" "${DESTINATION}/objects"
        else
            rsync -avch --progress "${INPUT}" "${DESTINATION}/${MEDIAID}/objects"
        fi
    fi
    _report -sb "Invoking makemetadata..."
    "${SCRIPT_PATH}/makemetadata" -m "${DESTINATION}/${MEDIAID}"
elif [[ "${PACKAGETYPE}" == "Digitized Film (MOV, MKV master files)" ]] ; then
    _report -sb "Invoking restructureSIP..."
    "${SCRIPT_PATH}/restructureSIP" -m "${MEDIAID}" -o "${DESTINATION}" -f "${INPUT}"
    _report -sb "Invoking makemetadata..."
    "${SCRIPT_PATH}/makemetadata" -m "${DESTINATION}/${MEDIAID}" # make metadata files, including MD5 file if not present
    _report -sb "Invoking makeH264..."
    "${SCRIPT_PATH}/makeH264" "${DESTINATION}/${MEDIAID}"
elif [[ "${PACKAGETYPE}" == "Digitized Analog Video (vrecord package)" ]] ; then
    _report -sb "Invoking restructureSIP..."
    "${SCRIPT_PATH}/restructureSIP" -m "${MEDIAID}" -o "${DESTINATION}" -v "${INPUT}"
    _report -sb "Invoking makemetadata..."
    "${SCRIPT_PATH}/makemetadata" -m "${DESTINATION}/${MEDIAID}" # make metadata files, including MD5 file if not present
    _report -sb "Invoking makeH264..."
    "${SCRIPT_PATH}/makeH264" "${DESTINATION}/${MEDIAID}"
elif [[ "${PACKAGETYPE}" == "Transferred DV (MOV, MKV master files)" ]] ; then
    _report -sb "Invoking restructureSIP..."
    "${SCRIPT_PATH}/restructureSIP" -m "${MEDIAID}" -o "${DESTINATION}" -d "${INPUT}"
    _report -sb "Invoking makemetadata..."
    "${SCRIPT_PATH}/makemetadata" -mq "${DESTINATION}/${MEDIAID}" # make metadata files, including MD5 and QCTools files  if not present
    _report -sb "Invoking makeH264..."
    "${SCRIPT_PATH}/makeH264" "${DESTINATION}/${MEDIAID}"
elif [[ "${PACKAGETYPE}" == "Great Migration (various formats)" ]] ; then
    _report -g "Moving Great Migration package as-is to specified destination..."
    rsync -avch --progress "${INPUT}" "${DESTINATION}/${MEDIAID}/objects"
    "${SCRIPT_PATH}/makemetadata" -m "${DESTINATION}/${MEDIAID}" # make metadata files, including MD5 file if not present
elif [[ "${PACKAGETYPE}" == "Other/Unknown" ]] ; then
    _report -sb "Invoking restructureSIP..."
    "${SCRIPT_PATH}/restructureSIP" -m "${MEDIAID}" -o "${DESTINATION}" -u "${INPUT}"
    _report -sb "Invoking makemetadata..."
    "${SCRIPT_PATH}/makemetadata" -m "${DESTINATION}/${MEDIAID}" # make metadata files, including MD5 file if not present
    _report -sb "Invoking makeH264..."
    "${SCRIPT_PATH}/makeH264" "${DESTINATION}/${MEDIAID}"
fi

# clean up original files if requested - hardcoded OFF during testing. Need to make sure that this also notices rsync errors from other microservices invoked within this script, specifically restructureSIP and restructureDPX.
#if [[ "${CLEANUPDECISION}" == "Remove source files after successful ingest" ]] ; then
#    [[ "${RSYNC_ERROR_COUNT}" -gt "0" ]] ; then
#        _report -rt  "Cancelling requested deletion of source file, not looking safe."
#    else
#        _report -g "Removing the source file from ${INPUT} as requested..."
#        _run rm -f "${INPUT}"
#    fi
#fi

# log script ending
_log -e
_writelog "END TIME" "$(date +%FT%T)"

_report -gs "ingestfile process done. Your AIP can be found at ${DESTINATION}/${MEDIAID}"
echo