Ci-dessous, les différences entre deux révisions de la page.
Les deux révisions précédentesRévision précédente | |||
04-linux:40-rsync [2016/11/16 12:09] – Roge | 04-linux:40-rsync [2017/10/06 23:38] (Version actuelle) – modification externe 127.0.0.1 | ||
---|---|---|---|
Ligne 1: | Ligne 1: | ||
+ | ====== rsync & ssh ====== | ||
+ | |||
+ | Pour réaliser le backup d'un dossier vers un dossier d'une autre machine il faut : | ||
+ | - Disposer d'un accès à la machine distante via SSH ([[04-linux: | ||
+ | |||
+ | - [[04-linux: | ||
+ | - Utiliser les scripts de backup. | ||
+ | |||
+ | ===== Le script bash de backup principal (générique) ===== | ||
+ | Ce script n'est rien d' | ||
+ | |||
+ | <file bash sshDirSynchro.sh> | ||
+ | #!/bin/bash | ||
+ | # sshDirSynchro.sh | ||
+ | # | ||
+ | # | ||
+ | # It is just an Rsync wrapper. | ||
+ | # | ||
+ | # V0.1 - 2014-08-17 | ||
+ | # V0.2 - 2014-08-18 | ||
+ | # Suppression du rapport de syncho intégré | ||
+ | # Suppression de messages | ||
+ | # V0.3 - 2014-08-18 | ||
+ | # l' | ||
+ | # l' | ||
+ | # l' | ||
+ | # lors d'une copie depuis Windows (on fixe les droits chmod=u+rwx). | ||
+ | |||
+ | TIME_START_1970=$(date +%s) | ||
+ | |||
+ | # | ||
+ | # Define usage | ||
+ | # | ||
+ | |||
+ | usage() { | ||
+ | cat << | ||
+ | |||
+ | sshDirSynchro.sh: | ||
+ | | ||
+ | Usage: sshDirSynchro.sh --backup|--restore -ld < | ||
+ | ] | ||
+ | |||
+ | Mandatory parameters: | ||
+ | --backup | ||
+ | copy from local to remote | ||
+ | --restore | ||
+ | copy from remote to local | ||
+ | -ld, --local-dir | ||
+ | Local directory from where the data are copied | ||
+ | -rh, --remote-host | ||
+ | Remote host to where the data are copied | ||
+ | -ru, --remote-user | ||
+ | Authorized user for the remote host | ||
+ | -rd, --remote-dir | ||
+ | Remote directory to where the data are copied | ||
+ | |||
+ | Optionnal parameters: | ||
+ | -h, --help | ||
+ | Display usage information. | ||
+ | -p, --ssh-port | ||
+ | Default port is 22. | ||
+ | -bwl, --bandwidth-limit | ||
+ | Embeded " | ||
+ | -ksd, --kill-sleep-delay | ||
+ | Permit to kill the script after a user defined delay. | ||
+ | Exemple: -ksd " | ||
+ | See man sleep for delay format. | ||
+ | -lf, --log-file | ||
+ | If no log file is set, the report is writen on stdout. | ||
+ | If -lf is set a log file is created for the report. | ||
+ | --delete | ||
+ | rsync delete extraneous files from dest dirs | ||
+ | --chmod | ||
+ | ensure chmod=u+rwx on remote host | ||
+ | --inconv | ||
+ | convert chrset from Windows to Linux | ||
+ | --check | ||
+ | compare local and remote dir content | ||
+ | |||
+ | *** Caution: | ||
+ | This script use several ssh commands in batch mode. To avoid password asking ssh keys must be installed. | ||
+ | See man ssh-keygen | ||
+ | | ||
+ | _usage_statement_ | ||
+ | } | ||
+ | |||
+ | # | ||
+ | # Analyse les arguments de la ligne de commande. | ||
+ | # | ||
+ | SSH_PORT=22 | ||
+ | # echo -e " | ||
+ | while [[ $# > 0 ]] | ||
+ | do | ||
+ | # echo -e "$# - OPTION: $1" | ||
+ | case $1 in | ||
+ | --backup) | ||
+ | if [ " | ||
+ | then | ||
+ | echo -e " | ||
+ | exit 1 | ||
+ | fi | ||
+ | SYNC_OPTION=" | ||
+ | shift | ||
+ | ;; | ||
+ | --restore) | ||
+ | if [ " | ||
+ | then | ||
+ | echo -e " | ||
+ | exit 1 | ||
+ | fi | ||
+ | SYNC_OPTION=" | ||
+ | shift | ||
+ | ;; | ||
+ | -ld|--local-dir) | ||
+ | LOCAL_WORKING_DIR=$2; | ||
+ | shift 2 | ||
+ | ;; | ||
+ | -rh|--remote-host) | ||
+ | REMOTE_HOST=$2 | ||
+ | shift 2 | ||
+ | ;; | ||
+ | -ru|--remote-user) | ||
+ | REMOTE_USER=$2 | ||
+ | shift 2 | ||
+ | ;; | ||
+ | -rd|--remote-dir) | ||
+ | REMOTE_WORKING_DIR=$2 | ||
+ | shift 2 | ||
+ | ;; | ||
+ | -p|--ssh-port) | ||
+ | SSH_PORT=$2 | ||
+ | shift 2 | ||
+ | ;; | ||
+ | -bwl|--bandwidth-limit) | ||
+ | BAND_WIDTH_LIMIT=$2 | ||
+ | shift 2 | ||
+ | ;; | ||
+ | -ksd|--kill-sleep-delay) | ||
+ | KILL_SLEEP_DELAY=$2 | ||
+ | shift 2 | ||
+ | ;; | ||
+ | -lf|--log-file) | ||
+ | LOGFILE=" | ||
+ | shift | ||
+ | ;; | ||
+ | --delete) | ||
+ | DELETE_OPTION=" | ||
+ | shift | ||
+ | ;; | ||
+ | --chmod) | ||
+ | CHMOD_OPTION=" | ||
+ | shift | ||
+ | ;; | ||
+ | --iconv) | ||
+ | ICONV_OPTION=" | ||
+ | shift | ||
+ | ;; | ||
+ | --check) | ||
+ | CHECK=$1 | ||
+ | shift | ||
+ | ;; | ||
+ | -h|--help) | ||
+ | usage | ||
+ | exit 0 | ||
+ | ;; | ||
+ | *) # Unknown option | ||
+ | echo -e "Error : Unknown option $1\n" | ||
+ | usage | ||
+ | exit -1 | ||
+ | ;; | ||
+ | esac | ||
+ | done | ||
+ | |||
+ | [ " | ||
+ | [ " | ||
+ | [ ${REMOTE_HOST} ] || echo -e " | ||
+ | [ ${REMOTE_USER} ] || echo -e " | ||
+ | [ ${REMOTE_WORKING_DIR} ] || echo -e " | ||
+ | |||
+ | if [ ! " | ||
+ | then | ||
+ | usage | ||
+ | exit -1 | ||
+ | fi | ||
+ | |||
+ | # | ||
+ | # Create a custom logger | ||
+ | # | ||
+ | |||
+ | if [ ${LOGFILE} ] | ||
+ | then | ||
+ | | ||
+ | #echo -e " | ||
+ | echo -e " | ||
+ | } | ||
+ | else | ||
+ | | ||
+ | #echo -e " | ||
+ | echo -e " | ||
+ | } | ||
+ | fi | ||
+ | |||
+ | # | ||
+ | # Test local diretory is ok | ||
+ | # | ||
+ | |||
+ | # if [ ! -d ${LOCAL_WORKING_DIR} ] | ||
+ | # then | ||
+ | # logprint | ||
+ | # exit -1 | ||
+ | # fi | ||
+ | |||
+ | # | ||
+ | # Test remote host is ok | ||
+ | # | ||
+ | |||
+ | if [ $LOGFILE ] | ||
+ | then | ||
+ | echo -e " | ||
+ | fi | ||
+ | |||
+ | # if ( ! ssh $REMOTE_USER@$REMOTE_HOST -p $SSH_PORT " | ||
+ | # then | ||
+ | # logprint | ||
+ | # exit -1 | ||
+ | # fi | ||
+ | |||
+ | # if ( ! ssh $REMOTE_USER@$REMOTE_HOST -p $SSH_PORT "cd $REMOTE_WORKING_DIR" | ||
+ | # then | ||
+ | # logprint | ||
+ | # exit -1 | ||
+ | # fi | ||
+ | |||
+ | # | ||
+ | # Lancement du kill de ce processus en tâche de fond | ||
+ | # pour limiter la durée d' | ||
+ | # | ||
+ | # Attention: | ||
+ | # la commande rsync s' | ||
+ | # l' | ||
+ | # il faut donc faire un "pkill -P $$" pour tuer l' | ||
+ | |||
+ | if [ ${KILL_SLEEP_DELAY} ] | ||
+ | then | ||
+ | | ||
+ | |||
+ | sleep $KILL_SLEEP_DELAY \ | ||
+ | && logprint | ||
+ | && logprint | ||
+ | && logprint | ||
+ | && pkill -P $$ & | ||
+ | # variante d' | ||
+ | # logprint | ||
+ | |||
+ | # On stocke le PID de process lancé en backgound pour pouvoir le tuer | ||
+ | | ||
+ | fi | ||
+ | |||
+ | # | ||
+ | # Recupération sur erreur | ||
+ | # | ||
+ | function traitement_erreur { | ||
+ | kill $KILL_ME_PID > /dev/null 2>&1 | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | # remote_report | ||
+ | exit -1 | ||
+ | } | ||
+ | trap traitement_erreur SIGHUP SIGINT SIGTERM SIGKILL | ||
+ | |||
+ | # | ||
+ | # Set rsync options | ||
+ | # | ||
+ | # options rsync: | ||
+ | # -a, --archive | ||
+ | # -v, --verbose | ||
+ | # -S, --sparse | ||
+ | # -P équivalent à --partial --progress | ||
+ | # | ||
+ | # | ||
+ | # | ||
+ | # | ||
+ | # --no-p | ||
+ | # --no-g | ||
+ | # --chmod=ugo=rwX | ||
+ | # --delete-excluded | ||
+ | # --exclude=PATTERN | ||
+ | # -s, --protect-args | ||
+ | # --iconv=ISO-8859-1, | ||
+ | |||
+ | # RSYNC_OPTIONS=" | ||
+ | # le -e \"ssh -p $SSH_PORT \" pose problème dans RSYNC_OPTIONS | ||
+ | # le --chmod=u=rwX semble nécessaire pour le Raspi | ||
+ | |||
+ | RSYNC_OPTIONS=" | ||
+ | [ ${CHMOD_OPTION} ] && RSYNC_OPTIONS=" | ||
+ | [ ${DELETE_OPTION} ] && RSYNC_OPTIONS=" | ||
+ | [ " | ||
+ | [ ${BAND_WIDTH_LIMIT} ] && RSYNC_OPTIONS=" | ||
+ | # | ||
+ | |||
+ | #logprint " | ||
+ | logprint " | ||
+ | if [[ ${SYNC_OPTION} = " | ||
+ | then | ||
+ | logprint " | ||
+ | logprint " | ||
+ | fi | ||
+ | if [[ ${SYNC_OPTION} = " | ||
+ | then | ||
+ | logprint " | ||
+ | logprint " | ||
+ | fi | ||
+ | logprint " | ||
+ | logprint " | ||
+ | |||
+ | |||
+ | # | ||
+ | # Build final command | ||
+ | # | ||
+ | |||
+ | RSYNC_COMMAND=" | ||
+ | [ " | ||
+ | [ " | ||
+ | [ ${LOGFILE} ] && RSYNC_COMMAND=" | ||
+ | |||
+ | logprint " | ||
+ | |||
+ | # !!!! eval introduces potential vulnerabilities and unexpected behavior situations so must be used !!! | ||
+ | |||
+ | eval ${RSYNC_COMMAND} | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | exit -2 | ||
+ | } | ||
+ | |||
+ | [ " | ||
+ | |||
+ | # | ||
+ | # On tue le process de kill (qu'il existe ou pas) | ||
+ | # afin qu'il n' | ||
+ | # | ||
+ | if [ ${KILL_SLEEP_DELAY} ] | ||
+ | then | ||
+ | # logprint | ||
+ | kill $KILL_ME_PID > /dev/null 2>&1 | ||
+ | fi | ||
+ | |||
+ | # | ||
+ | # fin des opération | ||
+ | # | ||
+ | TIME_END_1970=$(date +%s) | ||
+ | logprint | ||
+ | logprint " | ||
+ | |||
+ | </ | ||
+ | |||
+ | ===== Le script contrôle du backup (générique) ===== | ||
+ | |||
+ | |||
+ | Ce script compare des repertoires qui euvent être locaux et/ou distants. [[00-linux: | ||
+ | |||
+ | <file bash compareDirs.sh> | ||
+ | #!/bin/bash | ||
+ | # | ||
+ | # | ||
+ | # | ||
+ | # V0.1 - 2014-11-23 | ||
+ | # Missing: SSH port management | ||
+ | |||
+ | # | ||
+ | # Define usage | ||
+ | # | ||
+ | |||
+ | usage() { | ||
+ | cat << | ||
+ | |||
+ | compareDirs.sh compares the top level content of 2 local or remote directories using du. | ||
+ | | ||
+ | Usage: compareDirs.sh [[user@]host1: | ||
+ | | ||
+ | Note: SSH port management is missing. | ||
+ | |||
+ | _usage_statement_ | ||
+ | } | ||
+ | |||
+ | # | ||
+ | # Analyse les arguments de la ligne de commande. | ||
+ | # | ||
+ | |||
+ | echo -e " | ||
+ | |||
+ | [[ $# != 2 ]] && { | ||
+ | usage | ||
+ | exit -1 | ||
+ | } | ||
+ | |||
+ | function getDirFromParam { | ||
+ | | ||
+ | if [ $idx == 0 ] | ||
+ | then | ||
+ | echo $1 | ||
+ | else | ||
+ | echo ${1:$idx} | ||
+ | fi | ||
+ | } | ||
+ | |||
+ | function getHostFromParam { | ||
+ | | ||
+ | if [ $idx != 0 ] | ||
+ | then | ||
+ | ((idx--)) | ||
+ | echo ${1:0:$idx} | ||
+ | fi | ||
+ | } | ||
+ | |||
+ | function checkHostAndDir { | ||
+ | | ||
+ | | ||
+ | if [ $host ] | ||
+ | then | ||
+ | # echo "$host - $dir --> REMOTE" | ||
+ | if ( ! ssh $host " | ||
+ | then | ||
+ | | ||
+ | | ||
+ | elif ( ! ssh $host "cd $dir" > /dev/null 2>&1 ) | ||
+ | then | ||
+ | | ||
+ | | ||
+ | fi | ||
+ | | ||
+ | # echo " | ||
+ | [ -d " | ||
+ | echo -e "Error $dir must be a directory!" | ||
+ | | ||
+ | } | ||
+ | fi | ||
+ | | ||
+ | } | ||
+ | |||
+ | dir1=$(getDirFromParam $1) | ||
+ | host1=$(getHostFromParam $1) | ||
+ | dir2=$(getDirFromParam $2) | ||
+ | host2=$(getHostFromParam $2) | ||
+ | |||
+ | # echo -e "dir1 : $dir1 - host1 : $host1\ndir2 : $dir2 - host2 : $host2" | ||
+ | |||
+ | checkHostAndDir " | ||
+ | [ $? -eq 0 ] || exit -1 | ||
+ | checkHostAndDir " | ||
+ | [ $? -eq 0 ] || exit -1 | ||
+ | |||
+ | # | ||
+ | # VARS | ||
+ | # | ||
+ | |||
+ | tmp_report_1="/ | ||
+ | tmp_report_2="/ | ||
+ | |||
+ | # | ||
+ | # Recupération sur erreur | ||
+ | # | ||
+ | function traitement_erreur { | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | rm -f $tmp_report_1 $tmp_report_2 | ||
+ | exit 1 | ||
+ | } | ||
+ | trap traitement_erreur SIGHUP SIGINT SIGTERM SIGKILL | ||
+ | |||
+ | # | ||
+ | # fonction de bilan de synchro | ||
+ | # déclarée ici pour pouvoir être utilisée par le traitement d' | ||
+ | # | ||
+ | |||
+ | function local_report { | ||
+ | | ||
+ | echo $1 | sed ': | ||
+ | } | ||
+ | function stats_folder() { | ||
+ | # Attention : le nom du dossier peut contenir des blancs | ||
+ | # il faut donc le traiter entre "" | ||
+ | local folder=" | ||
+ | local items=`find " | ||
+ | local taille=0 | ||
+ | echo -e " | ||
+ | # echo -e " | ||
+ | if [ $items -gt 0 ]; then | ||
+ | taille=$(du -bc " | ||
+ | | ||
+ | | ||
+ | echo -e " | ||
+ | else | ||
+ | echo vide | ||
+ | fi | ||
+ | } | ||
+ | local curdir=$(pwd) | ||
+ | local dir=" | ||
+ | echo -e "LOCAL report:" | ||
+ | echo -e " | ||
+ | cd " | ||
+ | for f in * ; do | ||
+ | # Attention : le nom du dossier peut contenir des blancs | ||
+ | # il faut donc le passer entre "" | ||
+ | #echo "---f: $f" | ||
+ | [ -d " | ||
+ | done | ||
+ | echo -e " | ||
+ | stats_folder " | ||
+ | # echo "--- pwd: $(pwd) " | ||
+ | cd " | ||
+ | echo | ||
+ | } | ||
+ | |||
+ | function remote_report { | ||
+ | local dir=" | ||
+ | local host=" | ||
+ | ssh $host " | ||
+ | function numFormat { | ||
+ | echo \$1 | sed ': | ||
+ | } | ||
+ | function stats_folder() { | ||
+ | # Attention : le nom du dossier peut contenir des blancs | ||
+ | # il faut donc le traiter entre \" | ||
+ | local folder=\" | ||
+ | local items=\`find \" | ||
+ | local taille=0 | ||
+ | | ||
+ | if test \$items -ne 0; then | ||
+ | taille=\$(du -bc \" | ||
+ | taille_f=\$(numFormat \$taille) | ||
+ | items_f=\$(numFormat \$items) | ||
+ | echo -e \" | ||
+ | | ||
+ | echo vide | ||
+ | fi | ||
+ | } | ||
+ | |||
+ | dir=$(readlink -f " | ||
+ | echo \" | ||
+ | echo $dir | ||
+ | cd $dir | ||
+ | for folder in * ; do | ||
+ | # Attention : le nom du dossier peut contenir des blancs | ||
+ | # il faut donc le passer entre \" | ||
+ | [ -d \" | ||
+ | done | ||
+ | echo -e \" | ||
+ | stats_folder \" | ||
+ | echo | ||
+ | " | ||
+ | } | ||
+ | echo | ||
+ | |||
+ | # set -x | ||
+ | |||
+ | if [ $host1 ] | ||
+ | then | ||
+ | | ||
+ | else | ||
+ | | ||
+ | fi | ||
+ | |||
+ | if [ $host2 ] | ||
+ | then | ||
+ | | ||
+ | else | ||
+ | | ||
+ | fi | ||
+ | |||
+ | sdiff -atw 140 $tmp_report_1 $tmp_report_2 | ||
+ | rm -f $tmp_report_1 $tmp_report_2 | ||
+ | </ | ||
+ | |||
+ | ===== Un exemple de script de backup ===== | ||
+ | |||
+ | < | ||
+ | #!/bin/bash | ||
+ | |||
+ | N73SM=" | ||
+ | RASPI=" | ||
+ | BUREAU=" | ||
+ | |||
+ | DIR_SCRIPTS_BUREAU="/ | ||
+ | DIR_SCRIPTS_N73SM="/ | ||
+ | DIR_SCRIPTS_RASPI="/ | ||
+ | |||
+ | case `hostname` in | ||
+ | $N73SM) | ||
+ | # Scripts : N73SM --> Raspi | ||
+ | echo -e " | ||
+ | ./ | ||
+ | -rd $DIR_SCRIPTS_RASPI \ | ||
+ | -ld $DIR_SCRIPTS_N73SM | ||
+ | ;; | ||
+ | $BUREAU) | ||
+ | # Scripts : Bureau --> N73SM | ||
+ | echo -e " | ||
+ | | ||
+ | -rd $DIR_SCRIPTS_N73SM \ | ||
+ | -ld $DIR_SCRIPTS_BUREAU | ||
+ | |||
+ | # execution sur N73SM pour alimenter le Raspi | ||
+ | ssh roge@n73sm "cd / | ||
+ | ;; | ||
+ | *) # Unknown option | ||
+ | echo " | ||
+ | exit -1 | ||
+ | ;; | ||
+ | esac | ||
+ | </ | ||
+ | |||
+ | ===== Un exemple de rapport de backup : ===== | ||
+ | Le rapport suivant est produit par le script ci-dessus : | ||
+ | < | ||
+ | $ ./ | ||
+ | Backup Scripts : Bureau --> N73SM | ||
+ | 09:18:47: ------------------------------------------------------ | ||
+ | 09:18:47: Synchronisation | ||
+ | 09: | ||
+ | 09: | ||
+ | 09: | ||
+ | 09: | ||
+ | 09:18:47: ------------------------------------------------------ | ||
+ | 09:18:47: | ||
+ | 09:18:47: >> rsync -avS --no-p --no-g --delete-before -e "ssh -p 22" "/ | ||
+ | building file list ... done | ||
+ | deleting sshDirectorySynchro.sh | ||
+ | ./ | ||
+ | archive/ | ||
+ | archive/ | ||
+ | |||
+ | sent 1429 bytes received 121 bytes 3100.00 bytes/sec | ||
+ | total size is 175567 | ||
+ | 09:18:47: ******** Fin du traitment OK! ********** | ||
+ | 09:18:47: | ||
+ | |||
+ | -------------- | ||
+ | LOCAL report : | ||
+ | -------------- | ||
+ | LOCAL_WORKING_DIR: | ||
+ | - archive: 21 fichiers, 133K | ||
+ | - exemples: 6 fichiers, 24K - exemples: 6 fichiers, 24K | ||
+ | - restore: 18 fichiers, 96K - restore: 18 fichiers, 96K | ||
+ | - test: 1 fichiers, 4,0K - test: 1 fichiers, 4,0K | ||
+ | -------------------------- | ||
+ | Total: | ||
+ | - .: 57 fichiers, 329K | ||
+ | </ | ||
+ | |||
+ | |||
+ | ===== Autres solutions ===== | ||
+ | |||
+ | Tester [[https:// | ||
+ | |||