ctimer_fetch

#!/usr/bin/env bash
# Fetch files specified by path from remote site
#set -x

if [ ! -f "$1" ] || [ ! -f "$1" ]; then
  echo "Please provide host-named file with list of paths (or directory with such fiels) to rsync as command line argument."
  exit 1
fi

host=$(basename $1)
dir="./${host}_files_$(date +%Y-%m-%d_%H-%M)"

mkdir -p "${dir}/tree/"
mkdir -p "${dir}/_meta/"
rm -f "${dir}/_meta/paths.txt"

rsync -azh --files-from="$1" "${host}":/ "${dir}/tree/"

# copy files to flat folder and add numerical suffix
function safecp() {
  idx=1                                     # set the copy index to 1
  fn="${file##*/}"                          # destination file name (dfn) w/path stripped
  dfn="$fn"

  while [ -f "${dir}/${dfn}" ]; do          # test if $dfn exist in output_dir
    dfn="${fn%.*}_$((idx++)).${fn##*.}"     # if so, add copy index "_#" before extenison (increment until unique)
  done

  cp "$file" "${dir}/${dfn}"
  echo "${dfn} - ${file}" >> "${dir}/_meta/paths.txt"
}

# copy files to flat folder and add md5 hash suffix, if unique
function hashcp() {
  
  hash=$(md5sum -b "$file" | cut -f1 -d' ')

  if [[ ! ${hashes[*]} =~ $hash ]]; then
    hashes+=($hash)	
    fn="${file##*/}"
    dfn="${fn%.*}_${hash::8}.${fn##*.}"
    cp "$file" "${dir}/${dfn}"
    echo "${dfn} - ${file}" >> "${dir}/_meta/paths.txt"
  else
    echo "${file} DUPLICATE with ${hash}" >> "${dir}/_meta/paths.txt"
  fi

}

hashes=()

while read -d '' file; do
  hashcp </dev/null
done < <(find "${dir}/tree/" -type f -print0)

# printf "%s\n" "${hashes[@]}" >> "${dir}/_meta/paths.txt"

rm -rf "${dir}/tree/"

mv "$1" "${dir}"/_meta/"$host"