neo8000-imex

#!/bin/bash

source "neo8000-imex.conf" || exit 1

if [[ -z "$storage" ]]; then
    echo "ERROR: \$storage isn't set"
    exit 2
fi

transfers=0
label=0
option=""
pools=(Inc Full Diff Inc)
lockfile=/tmp/$0.lock
check_running=1
curpid=$$

trim() {
    echo $1|sed 's/^\s\+\|\s\+$//g';
}

unlock() {
    rm -f $lockfile
}

lock() {
    if [ -f $lockfile ]; then
        lastpid=$(<$lockfile)
        ps -p $lastpid
        if [ $? -eq 0 ]; then
            echo "ERROR: Script is already running!"
            exit 3
        else
            unlock
        fi
    fi
    echo $curpid > $lockfile
}

bacula_exec() {
    echo "$@" | bconsole -s -n > /dev/null;
}

# http://wiki.bacula.org/doku.php?id=faq#what_do_all_those_job_status_codes_mean
bacula_check_running() {
    running=$(echo "list jobs" | bconsole -s -n | tail -n 100 | awk -F\| '{print $9}' | sed -e 's/\s//g' | egrep 'R|C|F|j|M|m|p|c')
    if [ -n "$running" ]; then
        echo "ERROR: Bacula has currently running jobs, aborting!"
        exit 1
    fi
}

echo_f() {
	if [ -n "$jobsqueue" -a -n "$jobsdone" ]; then
		echo
		echo "-- $jobsdone/$jobsqueue -- $@ --";
	else
		echo
		echo "-- $@ --"
	fi
}

bconsole_update_retention() {
    echo_f "updating volumes retention"
    bconsole << END_OF_DATA
@output /dev/null
update volume
14
wait
quit
END_OF_DATA
    echo "done!"
}

bconsole_import() {
    bconsole << END_OF_DATA
@output /tmp/import-drive-$d.log
@time
label barcodes slots=$t drive=$d pool=${pools[$p]}
yes
wait
mount drive=$d slot=
wait
quit
END_OF_DATA
	((queue_finished++))
}

bacula_unmount_drives() {
    echo_f "unmounting drives"
    ((jobsdone++))
    for d in ${drives[@]}; do
        echo "unmounting drive-$d"
        bacula_exec "unmount drive=$d"
    done
}

bacula_mount_drives() {
    echo_f "mounting drives"
    ((jobsdone++))
    for d in ${drives[@]}; do
        echo "mounting drive-$d"
        bacula_exec "mount drive=$d slot="
    done
}

bacula_update() {
    echo_f "updating bacula"
    bacula_exec "update slots"
    echo "done!"
}

calc_transfers() {
    if [ $1 -lt $2 ]; then
        transfers=$1
    else
        transfers=$2
    fi
}

usage() {
    echo "usage: $0 [options] (--import|--export)"
    echo
    echo '    --import    - start import to bacula from mailslots'
    echo '    --export    - start export to bacula from mailslots'
    echo '    --label     - label volumes after import'
    echo '    --no-check  - skip check for running or waiting bacula jobs'
    echo '    -s          - path to storage device'
    echo '    -m          - path to mtx-changer script'
    echo '    -h          - outputs this message'
    exit
}

# Argument parsing
[[ $1 ]] || usage
while [[ $1 ]]; do
    case "$1" in
        '-s')          storage="$2"    ;;
        '--import')    option="import" ;;
        '--export')    option="export" ;;
        '--pools')     pools=($2)      ;;
        '--label')     label=1         ;;
        '--no-check')  check_running=0 ;;
        '-h'|'--help') usage           ;;
        '-m')          mtxchanger="$2" ;;
        -*)            echo "$0: Option \`$1' is not valid." ; exit 5 ;;
        *) ;;
    esac
    shift
done

if [ $check_running -ne 0 ]; then
    bacula_check_running
fi
lock

if [[ $option == 'import' || $label -gt 0 ]]; then
    bacula_update
    bacula_unmount_drives

    declare -a emptyslots=($($mtxchanger $storage listall | grep '^S:.*:E$' | awk -F: '{print $2}'))
    declare -a mailslots=($($mtxchanger $storage listall | grep '^I:.*:F:' | awk -F: '{print $2}'))

    calc_transfers ${#mailslots[*]} ${#emptyslots[*]}

    if [[ $option == 'import' && $label -gt 0 ]]; then
        jobqueue=3
    else
        jobqueue=2
    fi
    jobsdone=1
    maxslots=0
    if [[ $option == 'import' ]]; then
        echo_f "statistics"
        echo "Empty slots found: ${#emptyslots[*]}"
        echo "Mailslots used ${#mailslots[*]} from 15"

        if [ "${#emptyslots[*]}" -lt "${#mailslots[*]}" ]; then
            echo "WARNING: not enough free slots (free slots: ${#emptyslots[*]}, mailslots: ${#mailslots[*]})"
            maxslots=${#emptyslots[*]}
        else
            maxslots=${#mailslots[*]}
        fi

        echo_f "starting transfer"
        ((jobsdone++))
        mindex=0
        for e in ${emptyslots[@]}; do
            [[ $mindex -eq $maxslots ]] && break
            transfer=$(($mindex+1))
            echo "($transfer/$transfers) moving tape from ${mailslots[$mindex]} to $e"
            $mtxchanger $storage transfer ${mailslots[$mindex]} $e
            ((mindex++))
        done
    fi

    if [ $label -gt 0 ]; then
        echo_f "updating bacula volume names"
        ((jobsdone++))
        unlabeled=($(echo "update slots drive=0" | bconsole -s -n | \
            grep 'not found in catalog' | \
            sed 's/.*Slot=\([0-9]\+\).*/\1/g'))
        chunks=()
        ci=0
        rest=`((${#unlabeled[*]} % ${#drives[*]}))`
        for drive in ${drives[@]}; do
            div=`echo "${#unlabeled[*]} / ${#drives[*]}" | bc -l | xargs printf '%1.0f'`
            if [ $ci -eq ${#drives[*]} ]; then
                chunks[$ci]=`(($div+$rest))`
            else
                chunks[$ci]=$div
            fi
            ((ci++))
        done

	# FIXME no correct handling for chunks
        newtapes=($(echo "update slots drive=0" | bconsole -s -n | \
            grep 'not found in catalog' | \
            sed 's/.*Slot=\([0-9]\+\).*/\1/g' | \
            xargs -n 4 | \
            tr ' ' ','))
        d=0
	p=0
        echo "redirecting output to /tmp/import-drive-{0,1,2,3}.log"
        for t in ${newtapes[@]}; do
            touch /tmp/import-drive-$d.log

            echo "* Started import to bacula pool=${pools[$p]} with drive=$d"
            bconsole_import &
            ((d++))
            ((p++))
            if [ $p -ge ${#pools[*]} ]; then
                p=0
            fi
            [[ $d -gt $d ]] && break
        done

    	echo 'Waiting for completed label jobs'
	wait
    fi

    bacula_update

    bconsole_update_retention

    if [[ $option == 'import' ]]; then
        echo_f "import finished"
    else
        echo_f "labeling finished"
    fi
fi

if [[ $option == 'export' ]]; then
    jobqueue=4
    jobsdone=1
    full_volume_query="SELECT volumename,slot FROM media WHERE inchanger=1 AND volstatus='Full' ORDER BY lastwritten"

    bacula_update
    bacula_unmount_drives

    {
        IFS=$'\n'
        declare -a fullslots=($($mtxchanger $storage listall | grep '^S:.*:F'))
        declare -a fullvolumes=($(psql -d bacula -c "$full_volume_query" --tuples-only|grep 'ST'))
        declare -a emptymailslots=($($mtxchanger $storage listall | grep '^I:.*:E$' | awk -F: '{print $2}'))
    }

    calc_transfers ${#fullslots[*]} ${#emptymailslots[*]}

    echo_f "statistics"
    echo "Full slots: ${#fullslots[*]}"
    echo "Full volumes: ${#fullvolumes[*]}"
    echo "Empty mailslots: ${#emptymailslots[*]}"

    echo_f "starting transfer"
    ((jobsdone++))
    fvindex=0
    for mslot in ${emptymailslots[@]}; do
        transfer=$(($fvindex+1))
        vol=${fullvolumes[$fvindex]}
        label=`trim "$(echo $vol|awk -F\| '{print $1}')"`
        slot=`trim "$(echo $vol|awk -F\| '{print $2}')"`
        echo "($transfer/$transfers) moving full volume $label from $slot to $mslot"
        $mtxchanger $storage transfer $slot $mslot
        [[ $fvindex -gt ${#emptymailslots[*]} ]] && break
        ((fvindex++))
    done

    bacula_mount_drives

    bacula_update

    echo_f "export finished"
fi

unlock