This script can be configured in cron job to scheduled to run hourly and i will create one repo every week of the day and do differential backups every day. New repo means new full backup. save as .sh shell script and schedule the path of the file in the crontab. I would like to run these backups from a out of cluster node to safe keeping locally even though they uploaded to S3.
#!/bin/bash
#Create New repository Every Week to take Full backups Weekly. When #New Repo is used it take full backups otherwise Incremental
# Choose your day for Full backup
# compress backups to save space.
#Merge all small differential backups to the full backups which is #taken per repair first time.
#upload to S3 directly. you no need to set access credential instead, #you can use IAM role assigned to the EC2 instance role.
if [ "$(date '+%A')" == "Thursday" ]; then mkdir -p "/data/CB-Backups/$(date +"%m_%d_%Y")";fi
# Set Parameters
ARCHIVE="$(ls -td -- /data/CB-Backups/* | head -n 1)"
REPO=CBCluster
cluster=cluster_endpoint:8091
USERNAME=admin_username
PASSWORD=**********
THREADS=4
RESTOREPOINTS=24
CBBACKUPMGR=/opt/couchbase/bin/cbbackupmgr
BACKUPREGEX="[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}_[0-9]{2}_[0-9]{2}.[0-9]{9}Z"
if [ "$(date '+%A')" == "Thursday" ] && [ -d "$ARCHIVE" ]; then
CMD="${CBBACKUPMGR} config --archive ${ARCHIVE} --repo CBCluster"
echo -e "Creating New Repo..\nCommand: ${CMD}"
$CMD
fi
# Running backup
CMD="${CBBACKUPMGR} backup --archive ${ARCHIVE} --repo ${REPO} --cluster ${cluster} --username ${USERNAME} --password ${PASSWORD} --threads ${THREADS}"
echo -e "Running backup...\nCommand: ${CMD}"
$CMD
# Compacting the backup
BACKUPLIST=$(${CBBACKUPMGR} list --archive ${ARCHIVE} --repo ${REPO} | awk '{print $NF}' | grep -E ${BACKUPREGEX})
LASTBACKUP=$(echo "${BACKUPLIST}" | sed '$!d')
CMD="${CBBACKUPMGR} compact --archive ${ARCHIVE} --repo ${REPO} --backup ${LASTBACKUP}"
echo -e "Compacting the backup...\nCommand: ${CMD}"
$CMD
# Merging old backups
COUNT=$(echo "${BACKUPLIST}" | wc -l)
if [ "$COUNT" -gt "$RESTOREPOINTS" ]; then
START=$(echo "${BACKUPLIST}" | sed -n 1p)
END=$(echo "${BACKUPLIST}" | sed -n $((1+COUNT-RESTOREPOINTS))p)
CMD="${CBBACKUPMGR} merge --archive ${ARCHIVE} --repo ${REPO} --start ${START} --end ${END}"
echo -e "Merging old backups...\nCommand: ${CMD}"
$CMD
fi
# Upload content to S3
CMD="aws s3 sync /data/CB-Backups s3://bucket-name-env-backups/Couchbase/CB-Backups/"
echo -e "uploading backups to S3 ..\nCommand:${CMD}"
$CMD