copy_to_storage.sh

#!/bin/sh
# Move old archive files to permanent storage (S3 bucket).
#
# Optional parameter specifies date to move (default: yesterday)
#
# This must be run as wardenarchiver user (sudo -u wardenarchiver)

# TODO: edit according to your settings
local_dir=/data/warden_archive/archive
s3config=/data/warden_archive/s3cfg
bucket_name="warden-archive"
report_email=CHANGE_ME

echo "Script started at $(date)"

if ! [ -z "$1" ]; then
  # if parameter given, use it as date to copy
  file_to_copy="${1}.gz"
  file_to_remove="$(date -Idate -d "${1} - 7days").gz"
else
  # otherwise, copy file from yesterday
  file_to_copy="$(date -Idate -d "-1days").gz"
  file_to_remove="$(date -Idate -d "-8days").gz"
fi

echo "'$file_to_copy' will be copied to S3 storage, '$file_to_remove' will be removed from local disk."

# Copy the latest finished file to the storage
if s3cmd -c "$s3config" put --multipart-chunk-size-mb=200 -q "${local_dir}/${file_to_copy}" "s3://$bucket_name"
then
  echo "'$file_to_copy' successfully copied to S3 storage"
else
  echo "ERROR copying to S3 storage"
  me=$(hostname)
  /usr/sbin/sendmail -t <<EOF
To: $report_email
From: wardenarchiver@$me
Subject: [Warden archive] ERROR: Can't move file to S3 storage

There was an error when trying to copy $file_to_copy from the local archive to S3 storage. Please, look at logs and find out what is wrong.

Warden archiver

(generated by script '$0' at '$me')
EOF
fi

# Remove local files older than a week (if present)
if [ -e ${local_dir}/${file_to_remove} ] ; then
  # First, check that the file to remove is really present in the long-term storage
  # (and, just to be sure, that they both have the same size)
  echo "Check for file to be removed ($file_to_remove) on S3 storage ..."
  local_file_size=$(stat ${local_dir}/${file_to_remove} | sed -En 's/^ *Size: *([0-9]*).*$/\1/p')
  s3_file_size=$(s3cmd -c "$s3config" info s3://${bucket_name}/${file_to_remove} | sed -En 's/^ *File size: *([0-9]*).*$/\1/p')
  echo "S3 size: $s3_file_size ; local size: $local_file_size"
  if [ -z "$s3_file_size" -o "$s3_file_size" != "$local_file_size" ]
  then
    # The file is not there or has different size, something is wrong
    echo "ERROR: File to be removed ($file_to_remove) not found in storage (or has different size), not removing from local archive"
    me=$(hostname)
    /usr/sbin/sendmail -t <<EOF
To: $report_email
From: wardenarchiver@$me
Subject: [Warden archive] ERROR: Can't remove old file

I wanted to remove the file $file_to_remove from the local archive. It should already be stored in the long-term storage (S3), but it is not there (or has a wrong size). I will not remove anything for now. Please, find out what is wrong.

Warden archiver

(generated by script '$0' at '$me')
EOF
  else
    # It is OK, we can remove the local file
    echo "OK"
    rm "$local_dir/$file_to_remove" && echo "$file_to_remove removed from local archive" || echo "ERROR removing the local file"
  fi

else
  echo "'$file_to_remove' is not present in local archive - nothing to remove."
fi

echo -e "Script finished at $(date)\n"