Difference between revisions of "SweStore/swetrans arc"

From SNIC Documentation
Jump to: navigation, search
(Created page with "<pre>#!/bin/bash function error_quit { echo -e "$@" echo echo -e 'Usage:' echo -e 'swstrans from|to [-r] from to' echo -e '-r:\t potional; remove copied fi...")
 
(Swestore documentation moved)
(Tag: New redirect)
 
(3 intermediate revisions by 2 users not shown)
Line 1: Line 1:
<pre>#!/bin/bash
+
#REDIRECT[[Swestore Documentation Moved]]
 
 
function error_quit {
 
    echo -e "$@"
 
    echo
 
    echo -e 'Usage:'
 
    echo -e 'swstrans from|to [-r] from  to'
 
    echo -e '-r:\t potional; remove copied files'
 
    echo -e 'from:\t origin path; can include "*", "?" as wildcards'
 
    echo -e 'to:\t destination directory name ending with "/"'
 
    echo
 
    echo -e 'Example:'
 
    echo -e "swstrans to './run_1/fld*' JET/"
 
    echo -e 'Copies files fld* from local ./run_1/ to srm://srm.swegrid.se/snic/dnsturb/JET/'
 
    echo
 
    echo -e "swstrans from 'JET/fld*' ./run_1/"
 
    echo -e 'Copies files fld* from srm://srm.swegrid.se/snic/dnsturb/JET/ to local ./run_1/'
 
    exit 1
 
}
 
 
 
# parameters
 
srm_pth='srm://srm.swegrid.se/snic/'
 
srm_dom='dnsturb/'
 
swstr=$srm_pth$srm_dom
 
srm_cp='arccp'
 
srm_cp_opt=' --retries=3 --timeout=30'
 
srm_ls='arcls'
 
srm_rm='arcrm'
 
loc_rm='rm'
 
loc_mkdir='mkdirhier'
 
loc_ls='ls'
 
logfile='log_file'
 
errorfile='error_file'
 
awk='awk'
 
sed='sed'
 
 
 
# variables
 
remove=0
 
file_list=''
 
error_files=''
 
copied_files=''
 
 
 
# arguments
 
args=("$@")
 
argsnr=$#
 
 
 
# check arguments
 
# parameters number check
 
if [ $[argsnr] -lt 3 -o $[argsnr] -gt 4 ]; then
 
    error_quit 'Wrong arguments number!'
 
fi
 
 
 
# take paths
 
path_from="${args[$[argsnr-2]]}"
 
path_to="${args[$[argsnr-1]]}"
 
 
 
# separate directory and base names
 
base_from="${path_from##*/}"
 
dir_from="${path_from:0:${#path_from} - ${#base_from}}"
 
 
 
base_to="${path_to##*/}"
 
dir_to="${path_to:0:${#path_to} - ${#base_to}}"
 
 
 
# we don't overwrite any files, so base_to should be empty
 
if [ "${base_to}" != ""  ]; then
 
    error_msg='path_to should be directory name ending with "/"'
 
    error_quit ${error_msg}
 
fi
 
 
 
# change unix wildcards for awk
 
set -f
 
base_from=$(echo ${base_from}|$sed -e 's/\./\\./g' -e 's/\*/\.\*/g' -e 's/\?/\./g')
 
base_from="/^"$base_from"$/ {print}"
 
 
 
# first parameter check
 
case "${args[0]}" in
 
    from)
 
# check directories
 
# from
 
# create sestore path removing leading ./
 
dir_from=$swstr$(echo ${dir_from}| $sed -e 's/^\.//' -e 's/^\///')
 
`$srm_ls $dir_from 1>/dev/null 2>/dev/null`
 
if [ $? -ne 0 ]; then
 
    echo "Input directory: "${dir_from}" does not exist."| tee -a $logfile
 
    exit 1
 
fi
 
# to
 
if [ ! -d  ${dir_to} ]; then
 
    echo "Output directory: "${dir_to}" does not exist on local machine."| tee -a $logfile
 
    echo "Creating: "${dir_to}| tee -a $logfile
 
    `$loc_mkdir ${dir_to} 1>/dev/null`
 
    if [ $? -ne 0 ]; then
 
echo "Cannot create: "${dir_to}| tee -a $logfile
 
exit 1
 
    fi
 
elif [ -L ${dir_to}  ]; then
 
    echo "Output directory: "${dir_to}" is a symbolic link!"| tee -a $logfile
 
    exit 1
 
fi
 
# get file list
 
file_list=`$srm_ls $dir_from 2>/dev/null | $awk "$base_from"`
 
# file_list=$($srm_ls $dir_from 2>/dev/null | $awk '/f./ {print}')
 
# set rm, ls command
 
rm=$srm_rm
 
ls=$loc_ls
 
;;
 
    to)
 
# check directories
 
# from
 
if [ ! -d  ${dir_from} ]; then
 
    echo "Input directory: "${dir_from}" does not exist on local machine!"| tee -a $logfile
 
    exit 1
 
fi
 
# to
 
# create sestore path removing leading ./
 
dir_to=$swstr$(echo ${dir_to}| $sed -e 's/^\.//' -e 's/^\///')
 
`$srm_ls $dir_to 1>/dev/null 2>/dev/null`
 
if [ $? -ne 0 ]; then
 
    echo "Output directory: "${dir_to}" does not exist."| tee -a $logfile
 
    echo "It will be created by "${srm_cp}" during copying files."| tee -a $logfile
 
fi
 
# get file list
 
file_list=`$loc_ls $dir_from 2>/dev/null | $awk "$base_from"`
 
 
 
# set rm, ls command
 
rm=$loc_rm
 
ls=$srm_ls
 
;;
 
    *) error_quit 'Wrong option'
 
;;
 
esac
 
 
 
 
 
# check remove option
 
if [ $[argsnr] -eq 4 -a "${args[1]}" == "-r" ]; then
 
    remove=1
 
fi
 
 
 
# check file_list
 
if [ "$file_list" == "" ]; then
 
    echo 'List of files is empty; nothing to do'| tee -a $logfile
 
    exit 0
 
fi
 
 
 
# copy files
 
for file in ${file_list}; do
 
# we don't want to overwrite files, so check if the file exists
 
    `$ls $dir_to$file 1>/dev/null 2>/dev/null`
 
    if [ $? -ne 0 ]; then
 
echo 'Copying file: '$dir_from$file' to '$dir_to | tee -a $logfile
 
$srm_cp $srm_cp_opt $dir_from$file $dir_to 1>/dev/null 2> /dev/null
 
if [ $? -eq 0 ]; then
 
    echo 'Done: ' $dir_from$file `date +%c`| tee -a $logfile
 
    copied_files=$copied_files' '$file
 
else
 
    echo 'ERROR copying: '$dir_from$file' transfer problem.' | tee -a $logfile
 
    error_files=$error_files' '$file
 
fi
 
    else
 
echo 'ERROR copying: '$dir_from$file'; file: '$dir_to$file' exists. Skipping.' | tee -a $logfile
 
error_files=$error_files' '$file
 
    fi
 
done
 
 
 
# removing copied files
 
if [ $[remove] -eq 1 ]; then
 
    for file in $copied_files; do
 
echo 'Removing file: '$dir_from$file | tee -a $logfile
 
`$rm $dir_from$file 1>/dev/null 2>/dev/null`
 
if [ $? -eq 0 ]; then
 
    echo $dir_from$file ' removed' | tee -a $logfile
 
else
 
    echo 'ERROR removing file: '$dir_from$file | tee -a $logfile
 
fi
 
    done
 
fi
 
 
 
# output error list
 
if [ "$error_files" != "" ]; then
 
    echo 'Files not copied:' | tee -a $errorfile
 
    for file in $error_files; do
 
echo $dir_from$file | tee -a $errorfile
 
    done
 
fi
 
 
 
echo 'Done: ' `date +%c`| tee -a $logfile
 
 
 
exit 0</pre>
 

Latest revision as of 13:09, 8 February 2023