SweStore/swetrans arc

#!/bin/bash

function error_quit {

   echo -e "$@"
   echo
   echo -e 'Usage:'
   echo -e 'swstrans from|to [-r] from  to'
   echo -e '-r:\t potional; remove copied files'
   echo -e 'from:\t origin path; can include "*", "?" as wildcards'
   echo -e 'to:\t destination directory name ending with "/"'
   echo
   echo -e 'Example:'
   echo -e "swstrans to './run_1/fld*' JET/"
   echo -e 'Copies files fld* from local ./run_1/ to srm://srm.swegrid.se/snic/dnsturb/JET/'
   echo
   echo -e "swstrans from 'JET/fld*' ./run_1/"
   echo -e 'Copies files fld* from srm://srm.swegrid.se/snic/dnsturb/JET/ to local ./run_1/'
   exit 1

}

1. parameters

srm_pth='srm://srm.swegrid.se/snic/' srm_dom='dnsturb/' swstr=$srm_pth$srm_dom srm_cp='arccp' srm_cp_opt=' --retries=3 --timeout=30' srm_ls='arcls' srm_rm='arcrm' loc_rm='rm' loc_mkdir='mkdirhier' loc_ls='ls' logfile='log_file' errorfile='error_file' awk='awk' sed='sed'

1. variables

remove=0 file_list= error_files= copied_files=

1. arguments

args=("$@") argsnr=$#

1. check arguments
2. parameters number check

if [ $[argsnr] -lt 3 -o $[argsnr] -gt 4 ]; then

   error_quit 'Wrong arguments number!'

fi

1. take paths

path_from="${args[$[argsnr-2]]}" path_to="${args[$[argsnr-1]]}"

1. separate directory and base names

base_from="${path_from##*/}" dir_from="${path_from:0:${#path_from} - ${#base_from}}"

base_to="${path_to##*/}" dir_to="${path_to:0:${#path_to} - ${#base_to}}"

1. we don't overwrite any files, so base_to should be empty

if [ "${base_to}" != "" ]; then

   error_msg='path_to should be directory name ending with "/"'
   error_quit ${error_msg}

fi

1. change unix wildcards for awk

set -f base_from=$(echo ${base_from}|$sed -e 's/\./\\./g' -e 's/\*/\.\*/g' -e 's/\?/\./g') base_from="/^"$base_from"$/ {print}"

1. first parameter check

case "${args[0]}" in

   from)

1. check directories
2. from
3. create sestore path removing leading ./

dir_from=$swstr$(echo ${dir_from}| $sed -e 's/^\.//' -e 's/^\///') `$srm_ls $dir_from 1>/dev/null 2>/dev/null` if [ $? -ne 0 ]; then echo "Input directory: "${dir_from}" does not exist."| tee -a $logfile exit 1 fi

1. to

if [ ! -d ${dir_to} ]; then echo "Output directory: "${dir_to}" does not exist on local machine."| tee -a $logfile echo "Creating: "${dir_to}| tee -a $logfile `$loc_mkdir ${dir_to} 1>/dev/null` if [ $? -ne 0 ]; then echo "Cannot create: "${dir_to}| tee -a $logfile exit 1 fi elif [ -L ${dir_to} ]; then echo "Output directory: "${dir_to}" is a symbolic link!"| tee -a $logfile exit 1 fi

1. get file list

file_list=`$srm_ls $dir_from 2>/dev/null | $awk "$base_from"`

1. file_list=$($srm_ls $dir_from 2>/dev/null | $awk '/f./ {print}')
2. set rm, ls command

rm=$srm_rm ls=$loc_ls ;;

   to)

1. check directories
2. from

if [ ! -d ${dir_from} ]; then echo "Input directory: "${dir_from}" does not exist on local machine!"| tee -a $logfile exit 1 fi

1. to
2. create sestore path removing leading ./

dir_to=$swstr$(echo ${dir_to}| $sed -e 's/^\.//' -e 's/^\///') `$srm_ls $dir_to 1>/dev/null 2>/dev/null` if [ $? -ne 0 ]; then echo "Output directory: "${dir_to}" does not exist."| tee -a $logfile echo "It will be created by "${srm_cp}" during copying files."| tee -a $logfile fi

1. get file list

file_list=`$loc_ls $dir_from 2>/dev/null | $awk "$base_from"`

1. set rm, ls command

rm=$loc_rm ls=$srm_ls ;;

   *) error_quit 'Wrong option'

;; esac

1. check remove option

if [ $[argsnr] -eq 4 -a "${args[1]}" == "-r" ]; then

   remove=1

fi

1. check file_list

if [ "$file_list" == "" ]; then

   echo 'List of files is empty; nothing to do'| tee -a $logfile
   exit 0

fi

1. copy files

for file in ${file_list}; do

1. we don't want to overwrite files, so check if the file exists

   `$ls $dir_to$file 1>/dev/null 2>/dev/null`
   if [ $? -ne 0 ]; then

echo 'Copying file: '$dir_from$file' to '$dir_to | tee -a $logfile $srm_cp $srm_cp_opt $dir_from$file $dir_to 1>/dev/null 2> /dev/null if [ $? -eq 0 ]; then echo 'Done: ' $dir_from$file `date +%c`| tee -a $logfile copied_files=$copied_files' '$file else echo 'ERROR copying: '$dir_from$file' transfer problem.' | tee -a $logfile error_files=$error_files' '$file fi

   else

echo 'ERROR copying: '$dir_from$file'; file: '$dir_to$file' exists. Skipping.' | tee -a $logfile error_files=$error_files' '$file

   fi

done

1. removing copied files

if [ $[remove] -eq 1 ]; then

   for file in $copied_files; do

echo 'Removing file: '$dir_from$file | tee -a $logfile `$rm $dir_from$file 1>/dev/null 2>/dev/null` if [ $? -eq 0 ]; then echo $dir_from$file ' removed' | tee -a $logfile else echo 'ERROR removing file: '$dir_from$file | tee -a $logfile fi

   done

fi

1. output error list

if [ "$error_files" != "" ]; then

   echo 'Files not copied:' | tee -a $errorfile
   for file in $error_files; do

echo $dir_from$file | tee -a $errorfile

   done

fi

echo 'Done: ' `date +%c`| tee -a $logfile

exit 0