#!/bin/bash
#ftp address and login info
IP=192.168.253.2
USER=eix
PSWD=eix
#IP=128.192.151.162
#USER=jack
#PSWD=123456
#large file size defination(100M)
LARGE_FILE_SIZE=1000000
#log file
LOG_FILE_PUT=$HOME/log/ftp.put.`date -u +"%Y%m%d"`.log
LOG_FILE_GET=$HOME/log/ftp.get.`date -u +"%Y%m%d"`.log
#remote file and directory defination
#REMOTE_FILE_DOWN_DIR=$HOME/spool/downfile
REMOTE_FILE_DOWN_DIR=/downfile
REMOTE_FILE_UP_DIR=/upfile
REMOTE_OK_FILE_LIST=ok_file_list.remote
REMOTE_FILE_LIST=file_list.remote
#local file directory defination
LCOAL_FILE_DOWN_DIR=$HOME/spool/downfile
LCOAL_FILE_UP_DIR=$HOME/spool/upfile
LOCAL_FILE_LIST=file_list.local
#directory defination for get file from ftp server
LOCAL_FILE_LIST_TO_GET=file_list_to_get.local
LOCAL_FILE_LIST_TO_REGET=file_list_to_reget.local
#directory defination for put file from ftp server
LOCAL_FILE_LIST_TO_PUT=file_list_to_put.local
LOCAL_FILE_LIST_TO_REPUT=file_list_to_reput.local
#file list for del(size error or md5 error)
LOCAL_FILE_LIST_TO_DEL_FOR_SIZE_ERR=file_list_to_del_for_size_err.local
LOCAL_FILE_LIST_TO_DEL_FOR_MD5_ERR=file_list_to_del_for_md5_err.local
print_title(){
echo "=============RunTime is [ `date +\"%Y-%m-%d %T\"` ], "$1==============>> $LOG_FILE
}
print_space(){
echo >> $LOG_FILE
}
#check file by md5
#argc: @filename
check_file_md5(){
_filename=$1
#produce md5
_md5=`md5sum $_filename | awk -F" " '{print $1}'`
echo "cocaculate md5 for file "$_filename": "$_md5 >> $LOG_FILE
#read md5 from ok file
md5=`cat $_filename.ok | awk -F"|" '{print $2}'`
echo "read md5 from file "$_filename": "$md5 >> $LOG_FILE
#compare md5
if [[ $_md5 != $md5 ]];then
echo "md5 not equal" >> $LOG_FILE
return 1
fi
}
#check file by size
check_file_size(){
_filename=$1
#concaculate file size
_filesize=`wc -c < $_filename`
echo "concaculate size for file "$_filename": "$_filesize >> $LOG_FILE
#add size from ok file
filesize=`head -1 $_filename.ok | awk -F"|" '{print $1}'`
echo "read size from file "$_filename": "$filesize >> $LOG_FILE
#compare file size
if [[ $filesize != $_filesize ]];then
echo "size not equal" >> $LOG_FILE
return 1
fi
}
#check file
check_file(){
_filename=$1
#check file by size
if check_file_size $_filename;then
echo "check file size success" >> $LOG_FILE
else
echo "check file size failure" >> $LOG_FILE
#compare the error file size is bigger than the size defined by LARGE_FILE_SIZE
_filesize=`wc -c < $_filename`
if [[ $_filesize -lt $LARGE_FILE_SIZE ]];then
echo "file size: "$_filesize" is lower than lage file defined size :"$LARGE_FILE_SIZE" need to delete!" >> $LOG_FILE
echo $_filename >> $LOCAL_FILE_LIST_TO_DEL_FOR_SIZE_ERR
fi
return 1
fi
#check file by md5
if check_file_md5 $_filename;then
echo "check file md5 success" >> $LOG_FILE
else
echo "check file md5 failure" >> $LOG_FILE
#record file error(name|err_md5|right_md5)
echo "file "$_filename " md5 check error, net to delete!" >> $LOG_FILE
echo $_filename >> $LOCAL_FILE_LIST_TO_DEL_FOR_MD5_ERR
return 1
fi
}
#check argc
if [ $# != 1 ]
then
echo "Usage: $0 get|put " >> &2
exit 1
fi
if [[ $1 = "get" ]];then
#login ftp server to get *.ok files
print_title "login ftp server:$IP"
print_space
ftp -i -v -n $IP <<!
user $USER $PSWD
cd $REMOTE_FILE_DOWN_DIR
binary
prompt off
#get ok file list, file size info
lcd $LCOAL_FILE_DOWN_DIR
mget *.ok
#logout ftp sever
bye
!
#check ok files exist or not
cd $LCOAL_FILE_DOWN_DIR
count=`ls *.ok | wc -l`
if [ $count -eq 0 ];then
echo "ok file not exists, their is no remote files to download" >> $LOG_GET_FILE
exit 1
fi
#---------------------------------------------------------------------------------
# produce file list to get
#---------------------------------------------------------------------------------
#get ok file list
ls *.ok > $REMOTE_OK_FILE_LIST
#get remote file list to local file(delete *.ok)
cat $REMOTE_OK_FILE_LIST | sed 's/\.[^\.]\+$//' > $REMOTE_FILE_LIST && rm -rf tmp
print_title "remote file list:"
cat $REMOTE_FILE_LIST >> $LOG_GET_FILE
print_space
#get local file(which had downloaded from ftp server) list
ls -l | sed "1d" | grep -Ev *.remote\|*.local\|*.ok | awk -F" " '{printf("%s\n", $9)}' > $LCOAL_FILE_DOWN_DIR/$LOCAL_FILE_LIST
print_title "local file list(had downloaded): "
cat $LCOAL_FILE_DOWN_DIR/$LOCAL_FILE_LIST >> $LOG_GET_FILE
print_space
#compare local file list and remote ok file list, prouce file list which is going to get
grep -v -wvf $LOCAL_FILE_LIST $REMOTE_FILE_LIST > $LCOAL_FILE_DOWN_DIR/$LOCAL_FILE_LIST_TO_GET
print_title "remote files are going to down:"
cat $LCOAL_FILE_DOWN_DIR/$LOCAL_FILE_LIST_TO_GET >> $LOG_GET_FILE
print_space
#delete local tmp file
rm -rf $LCOAL_FILE_DOWN_DIR/$REMOTE_FILE_LIST $LCOAL_FILE_DOWN_DIR/$LOCAL_FILE_LIST
#---------------------------------------------------------------------------------
# produce file list to reget
#---------------------------------------------------------------------------------
#merge remote ok file list info to one local file
cat $REMOTE_OK_FILE_LIST | while read file
do
name=`echo $file | sed 's/\.[^\.]\+$//'`
size=`head -1 $file | awk -F"|" '{print $1}'`
echo $name"|"$size >> $REMOTE_FILE_LIST
done
print_title "remote file list info(name|size):"
cat $REMOTE_FILE_LIST >> $LOG_GET_FILE
print_space
#get local file(which had downloaded from ftp server) list info(name|size)
ls -l | sed "1d" | grep -Ev *.remote\|*.local\|*.ok | awk -F" " '{printf("%s|%s\n", $9, $5)}' > $LCOAL_FILE_DOWN_DIR/$LOCAL_FILE_LIST
print_title "local file list info(name|size): "
cat $LOCAL_FILE_LIST >> $LOG_GET_FILE
print_space
#compare local file list info and remote file list info, produce file list which is to reget
join -t "|" -j1 1 -j2 1 $LCOAL_FILE_DOWN_DIR/$LOCAL_FILE_LIST $LCOAL_FILE_DOWN_DIR/$REMOTE_FILE_LIST |\
awk -F"|" '{if($2 < $3) print $1}' > $LCOAL_FILE_DOWN_DIR/$LOCAL_FILE_LIST_TO_REGET
print_title "remote files are going to reget:"
cat $LCOAL_FILE_DOWN_DIR/$LOCAL_FILE_LIST_TO_REGET >> $LOG_GET_FILE
print_space
#delete local tmp file
rm -rf $LCOAL_FILE_DOWN_DIR/$REMOTE_FILE_LIST $LCOAL_FILE_DOWN_DIR/$LOCAL_FILE_LIST
#---------------------------------------------------------------------------------
# get remote files
#---------------------------------------------------------------------------------
#get remote files
cat $LOCAL_FILE_LIST_TO_GET | while read file
do
#test $file is null or not
if [ -z $file ];then
echo "no files to get!" >> $LOG_GET_FILE
break;
fi
print_title "login ftp server:$IP"
echo "begin to download file: "$file >> $LOG_GET_FILE
#login ftp server
ftp -i -v -n $IP <<!
user $USER $PSWD
cd $REMOTE_FILE_DOWN_DIR
binary
prompt off
#get file
get $file
#logout ftp sever
bye
!
echo "end to get file: "$file >> $LOG_GET_FILE
print_space
#check file
if check_file $file;then
echo "successfully check file: "$file >> $LOG_GET_FILE
else
#delete local files and continue
echo "check file failure: "$file >> $LOG_GET_FILE
print_space
#get next file
continue
fi
#success download file
echo "successfully get file: "$file >> $LOG_GET_FILE
print_space
#get next file
continue
done
#---------------------------------------------------------------------------------
# reget remote files
#---------------------------------------------------------------------------------
#reget remote files
cat $LOCAL_FILE_LIST_TO_REGET | while read fil
