[Bps-public-commit] postgresql_auto_backup_s3 branch master created. 79d9148e6501e806a9735a48aa60564fc6da5d2a
BPS Git Server
git at git.bestpractical.com
Wed Mar 9 14:46:03 UTC 2022
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "postgresql_auto_backup_s3".
The branch, master has been created
at 79d9148e6501e806a9735a48aa60564fc6da5d2a (commit)
- Log -----------------------------------------------------------------
commit 79d9148e6501e806a9735a48aa60564fc6da5d2a
Author: rajanski <rajanski at users.noreply.github.com>
Date: Sun Jan 25 19:58:27 2015 +0100
added comments
TODO:
-monthly, weekly rotation
-TESTING
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index 47d4dda..10e36d6 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -176,9 +176,12 @@ fi
# Delete daily backups $DAYS_TO_KEEP days old or more
#find $BACKUP_DIR -maxdepth 1 -mtime +$DAYS_TO_KEEP -name "*-daily" -exec rm -rf '{}' ';'
-##CAUTION! do not use this YET!
+#set $deldate to date minus $DAYS_TO_KEEP
deldate=`date +"%Y%m%d" --date="$DAYS_TO_KEEP days ago"`
-#s3cmd ls $BACKUP_DIR | awk '{if ($1:0:10 == "$deldate") system("s3cmd info " $4)}'
+
+#remove leading bucket name from s3 object displayed by "s3cmd ls $bucketname"
+#compare first 8 chars of s3 object name (minus bucket name) with $deldate,
+#if first 8 chars of s3 object name::int <= $deldate then delete those s3 objects
s3cmd ls $BACKUP_DIR | awk -v bucket="$BACKUP_DIR" -v deldate="$deldate" '{gsub(bucket,"",$4);if (substr($4,0,9)<=deldate) system("s3cmd info " bucket$4)}'
#replace s3cmd info by s3cmd del
commit f3653f3505deefd105e553e2f1621f7fedac3f35
Author: rajanski <rajanski at users.noreply.github.com>
Date: Sun Jan 25 19:53:20 2015 +0100
daily rotation working, need to replace s3cmd "info" by "del" in line 182
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index 45fcd78..47d4dda 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -179,7 +179,7 @@ fi
##CAUTION! do not use this YET!
deldate=`date +"%Y%m%d" --date="$DAYS_TO_KEEP days ago"`
#s3cmd ls $BACKUP_DIR | awk '{if ($1:0:10 == "$deldate") system("s3cmd info " $4)}'
-s3cmd ls $BACKUP_DIR | awk -v bucket="$BACKUP_DIR" -v deldate="$deldate" '{gsub(bucket,"",$4);if (substr($4,0,8)<=deldate) system("s3cmd info " $4)}'
+s3cmd ls $BACKUP_DIR | awk -v bucket="$BACKUP_DIR" -v deldate="$deldate" '{gsub(bucket,"",$4);if (substr($4,0,9)<=deldate) system("s3cmd info " bucket$4)}'
#replace s3cmd info by s3cmd del
perform_backups "-daily"
commit 726d1132f363bb8fc527ce56fc44b5f3d968c798
Author: rajanski <rajanski at users.noreply.github.com>
Date: Sun Jan 25 19:41:12 2015 +0100
Update pg_backup_rotated.sh
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index 8d705a3..45fcd78 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -60,7 +60,7 @@ fi;
function perform_backups()
{
SUFFIX=$1
- FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y\%m\%d`$SUFFIX/"
+ FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y\%m\%d`$SUFFIX-"
#echo "Making backup directory in $FINAL_BACKUP_DIR"
commit 0c3f9b8b4d2b701dd2edac90c3a490dd08446fc7
Author: rajanski <rajanski at users.noreply.github.com>
Date: Sun Jan 25 19:40:07 2015 +0100
first draft rotation daily. do NOT use yet
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index f76d432..8d705a3 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -60,7 +60,7 @@ fi;
function perform_backups()
{
SUFFIX=$1
- FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y-\%m-\%d`$SUFFIX-"
+ FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y\%m\%d`$SUFFIX/"
#echo "Making backup directory in $FINAL_BACKUP_DIR"
@@ -173,7 +173,13 @@ fi
# DAILY BACKUPS
-# Delete daily backups 7 days old or more
-find $BACKUP_DIR -maxdepth 1 -mtime +$DAYS_TO_KEEP -name "*-daily" -exec rm -rf '{}' ';'
-
+# Delete daily backups $DAYS_TO_KEEP days old or more
+#find $BACKUP_DIR -maxdepth 1 -mtime +$DAYS_TO_KEEP -name "*-daily" -exec rm -rf '{}' ';'
+
+##CAUTION! do not use this YET!
+deldate=`date +"%Y%m%d" --date="$DAYS_TO_KEEP days ago"`
+#s3cmd ls $BACKUP_DIR | awk '{if ($1:0:10 == "$deldate") system("s3cmd info " $4)}'
+s3cmd ls $BACKUP_DIR | awk -v bucket="$BACKUP_DIR" -v deldate="$deldate" '{gsub(bucket,"",$4);if (substr($4,0,8)<=deldate) system("s3cmd info " $4)}'
+#replace s3cmd info by s3cmd del
+
perform_backups "-daily"
commit d07b0efb47b1e7eab083b8bdb5cecf5a55b05b27
Author: rajanski <rajanski at users.noreply.github.com>
Date: Sun Jan 25 18:06:22 2015 +0100
make final_backup_dir a filename, not a directory
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index 34f51fc..f76d432 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -60,7 +60,7 @@ fi;
function perform_backups()
{
SUFFIX=$1
- FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y-\%m-\%d`$SUFFIX/"
+ FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y-\%m-\%d`$SUFFIX-"
#echo "Making backup directory in $FINAL_BACKUP_DIR"
commit d94bb68c4554bffb78d4310f5e1f1d8305683e76
Author: rajanski <rajanski at users.noreply.github.com>
Date: Sun Jan 25 17:42:06 2015 +0100
Changed backup commands to send backup to s3 via stdout/stdin
Not tested yet
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index d6d22b0..34f51fc 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -62,12 +62,12 @@ function perform_backups()
SUFFIX=$1
FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y-\%m-\%d`$SUFFIX/"
- echo "Making backup directory in $FINAL_BACKUP_DIR"
+ #echo "Making backup directory in $FINAL_BACKUP_DIR"
- if ! mkdir -p $FINAL_BACKUP_DIR; then
- echo "Cannot create backup directory in $FINAL_BACKUP_DIR. Go and fix it!" 1>&2
- exit 1;
- fi;
+ #if ! mkdir -p $FINAL_BACKUP_DIR; then
+ # echo "Cannot create backup directory in $FINAL_BACKUP_DIR. Go and fix it!" 1>&2
+ # exit 1;
+ #fi;
###########################
@@ -91,11 +91,11 @@ function perform_backups()
for DATABASE in $SCHEMA_ONLY_DB_LIST
do
echo "Schema-only backup of $DATABASE"
-
- if ! pg_dump -Fp -s -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress; then
+
+ if ! pg_dump -Fp -s -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress --no-encrypt; then
echo "[!!ERROR!!] Failed to backup database schema of $DATABASE" 1>&2
else
- mv $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz
+ s3cmd mv $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz
fi
done
@@ -119,23 +119,22 @@ function perform_backups()
if [ $ENABLE_PLAIN_BACKUPS = "yes" ]
then
echo "Plain backup of $DATABASE"
-
- if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
- echo "[!!ERROR!!] Failed to produce plain backup database $DATABASE" 1>&2
- else
- mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
- fi
+
+ if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
+ echo "[!!ERROR!!] Failed to produce plain backup database schema of $DATABASE" 1>&2
+ else
+ s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
+ fi
fi
if [ $ENABLE_CUSTOM_BACKUPS = "yes" ]
then
echo "Custom backup of $DATABASE"
-
- if ! pg_dump -Fc -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" -f $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress; then
- echo "[!!ERROR!!] Failed to produce custom backup database $DATABASE"
- else
- mv $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress $FINAL_BACKUP_DIR"$DATABASE".custom
- fi
+ if ! pg_dump -Fc -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress --no-encrypt; then
+ echo "[!!ERROR!!] Failed to produce custom backup database schema of $DATABASE" 1>&2
+ else
+ s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress $FINAL_BACKUP_DIR"$DATABASE".custom
+ fi
fi
done
commit ba9a4761cc42cb84606a006fa0c128bbe2529bc9
Author: rajanski <rajanski at users.noreply.github.com>
Date: Sun Jan 25 17:28:50 2015 +0100
Update pg_backup.config
diff --git a/pg_backup.config b/pg_backup.config
index 4781280..5e97702 100644
--- a/pg_backup.config
+++ b/pg_backup.config
@@ -14,7 +14,9 @@ USERNAME=
# This dir will be created if it doesn't exist. This must be writable by the user the script is
# running as.
-BACKUP_DIR=/home/backups/database/postgresql/
+#BACKUP_DIR=/home/backups/database/postgresql/
+#change to s3 bucket
+BACKUP_DIR=s3://pg_backups/
# List of strings to match against in database name, separated by space or comma, for which we only
# wish to keep a backup of the schema, not the data. Any database names which contain any of these
commit 809a0119b7fd0c2d4038c3f6bae35ee2e33ca090
Author: rajanski <rajanski at users.noreply.github.com>
Date: Thu Jan 1 18:46:12 2015 +0100
Update README.md
diff --git a/README.md b/README.md
index 7572f9d..b6c2182 100644
--- a/README.md
+++ b/README.md
@@ -7,4 +7,4 @@ Contributors welcome!
Original code from https://wiki.postgresql.org/wiki/Automated_Backup_on_Linux.
-Will be adapted to work with s3cmd stdin pushing, there fore needs s3cmd >= s3cmd 1.5.0-alpha1 , also see http://s3tools.org/news
+Will be adapted to work with s3cmd stdin pushing, therefore needs s3cmd >= s3cmd 1.5.0-alpha1 , also see http://s3tools.org/news
commit 0d66e4296b3ebf0bff3a7d6a51d25185a191b1b5
Author: rajanski <rajanski at users.noreply.github.com>
Date: Thu Jan 1 18:45:22 2015 +0100
Update README.md
diff --git a/README.md b/README.md
index 6a0ee40..7572f9d 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,9 @@
postgresql_auto_backup_s3
=========================
-Configurable backup scripts to automate postgresql backups and push to s3 via stdin, plus rotation logic
+Configurable backup scripts to automate postgresql backups and push to s3 via stdin, plus rotation logic.
+
+Contributors welcome!
Original code from https://wiki.postgresql.org/wiki/Automated_Backup_on_Linux.
commit 9ca2c987f0b128cd502a47be382d534002804224
Author: rajanski <rajanski at users.noreply.github.com>
Date: Thu Jan 1 18:45:01 2015 +0100
Update README.md
diff --git a/README.md b/README.md
index 72af122..6a0ee40 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
postgresql_auto_backup_s3
=========================
-Backup Scripts to automate postgresql backups and push to s3 via stdin, plus rotation logic
+Configurable backup scripts to automate postgresql backups and push to s3 via stdin, plus rotation logic
Original code from https://wiki.postgresql.org/wiki/Automated_Backup_on_Linux.
commit e1781bf4df6015f984ae31a606c68a8797e9d9f5
Author: rajanski <rajanski at users.noreply.github.com>
Date: Thu Jan 1 18:44:33 2015 +0100
Update README.md
diff --git a/README.md b/README.md
index 0ae6eb9..72af122 100644
--- a/README.md
+++ b/README.md
@@ -5,4 +5,4 @@ Backup Scripts to automate postgresql backups and push to s3 via stdin, plus rot
Original code from https://wiki.postgresql.org/wiki/Automated_Backup_on_Linux.
-Will be adapted to wrk with s3cmd stdin pushing
+Will be adapted to work with s3cmd stdin pushing, there fore needs s3cmd >= s3cmd 1.5.0-alpha1 , also see http://s3tools.org/news
commit 33d686875b6f257d8833a63cd1dbd7741c137f4d
Author: rajanski <rajanski at users.noreply.github.com>
Date: Thu Jan 1 18:42:09 2015 +0100
Create pg_backup_rotated.sh
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
new file mode 100644
index 0000000..d6d22b0
--- /dev/null
+++ b/pg_backup_rotated.sh
@@ -0,0 +1,180 @@
+#!/bin/bash
+
+###########################
+####### LOAD CONFIG #######
+###########################
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ -c)
+ CONFIG_FILE_PATH="$2"
+ shift 2
+ ;;
+ *)
+ ${ECHO} "Unknown Option \"$1\"" 1>&2
+ exit 2
+ ;;
+ esac
+done
+
+if [ -z $CONFIG_FILE_PATH ] ; then
+ SCRIPTPATH=$(cd ${0%/*} && pwd -P)
+ CONFIG_FILE_PATH="${SCRIPTPATH}/pg_backup.config"
+fi
+
+if [ ! -r ${CONFIG_FILE_PATH} ] ; then
+ echo "Could not load config file from ${CONFIG_FILE_PATH}" 1>&2
+ exit 1
+fi
+
+source "${CONFIG_FILE_PATH}"
+
+###########################
+#### PRE-BACKUP CHECKS ####
+###########################
+
+# Make sure we're running as the required backup user
+if [ "$BACKUP_USER" != "" -a "$(id -un)" != "$BACKUP_USER" ] ; then
+ echo "This script must be run as $BACKUP_USER. Exiting." 1>&2
+ exit 1
+fi
+
+
+###########################
+### INITIALISE DEFAULTS ###
+###########################
+
+if [ ! $HOSTNAME ]; then
+ HOSTNAME="localhost"
+fi;
+
+if [ ! $USERNAME ]; then
+ USERNAME="postgres"
+fi;
+
+
+###########################
+#### START THE BACKUPS ####
+###########################
+
+function perform_backups()
+{
+ SUFFIX=$1
+ FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y-\%m-\%d`$SUFFIX/"
+
+ echo "Making backup directory in $FINAL_BACKUP_DIR"
+
+ if ! mkdir -p $FINAL_BACKUP_DIR; then
+ echo "Cannot create backup directory in $FINAL_BACKUP_DIR. Go and fix it!" 1>&2
+ exit 1;
+ fi;
+
+
+ ###########################
+ ### SCHEMA-ONLY BACKUPS ###
+ ###########################
+
+ for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
+ do
+ SCHEMA_ONLY_CLAUSE="$SCHEMA_ONLY_CLAUSE or datname ~ '$SCHEMA_ONLY_DB'"
+ done
+
+ SCHEMA_ONLY_QUERY="select datname from pg_database where false $SCHEMA_ONLY_CLAUSE order by datname;"
+
+ echo -e "\n\nPerforming schema-only backups"
+ echo -e "--------------------------------------------\n"
+
+ SCHEMA_ONLY_DB_LIST=`psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$SCHEMA_ONLY_QUERY" postgres`
+
+ echo -e "The following databases were matched for schema-only backup:\n${SCHEMA_ONLY_DB_LIST}\n"
+
+ for DATABASE in $SCHEMA_ONLY_DB_LIST
+ do
+ echo "Schema-only backup of $DATABASE"
+
+ if ! pg_dump -Fp -s -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress; then
+ echo "[!!ERROR!!] Failed to backup database schema of $DATABASE" 1>&2
+ else
+ mv $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz
+ fi
+ done
+
+
+ ###########################
+ ###### FULL BACKUPS #######
+ ###########################
+
+ for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
+ do
+ EXCLUDE_SCHEMA_ONLY_CLAUSE="$EXCLUDE_SCHEMA_ONLY_CLAUSE and datname !~ '$SCHEMA_ONLY_DB'"
+ done
+
+ FULL_BACKUP_QUERY="select datname from pg_database where not datistemplate and datallowconn $EXCLUDE_SCHEMA_ONLY_CLAUSE order by datname;"
+
+ echo -e "\n\nPerforming full backups"
+ echo -e "--------------------------------------------\n"
+
+ for DATABASE in `psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$FULL_BACKUP_QUERY" postgres`
+ do
+ if [ $ENABLE_PLAIN_BACKUPS = "yes" ]
+ then
+ echo "Plain backup of $DATABASE"
+
+ if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
+ echo "[!!ERROR!!] Failed to produce plain backup database $DATABASE" 1>&2
+ else
+ mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
+ fi
+ fi
+
+ if [ $ENABLE_CUSTOM_BACKUPS = "yes" ]
+ then
+ echo "Custom backup of $DATABASE"
+
+ if ! pg_dump -Fc -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" -f $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress; then
+ echo "[!!ERROR!!] Failed to produce custom backup database $DATABASE"
+ else
+ mv $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress $FINAL_BACKUP_DIR"$DATABASE".custom
+ fi
+ fi
+
+ done
+
+ echo -e "\nAll database backups complete!"
+}
+
+# MONTHLY BACKUPS
+
+DAY_OF_MONTH=`date +%d`
+
+if [ $DAY_OF_MONTH -eq 1 ];
+then
+ # Delete all expired monthly directories
+ find $BACKUP_DIR -maxdepth 1 -name "*-monthly" -exec rm -rf '{}' ';'
+
+ perform_backups "-monthly"
+
+ exit 0;
+fi
+
+# WEEKLY BACKUPS
+
+DAY_OF_WEEK=`date +%u` #1-7 (Monday-Sunday)
+EXPIRED_DAYS=`expr $((($WEEKS_TO_KEEP * 7) + 1))`
+
+if [ $DAY_OF_WEEK = $DAY_OF_WEEK_TO_KEEP ];
+then
+ # Delete all expired weekly directories
+ find $BACKUP_DIR -maxdepth 1 -mtime +$EXPIRED_DAYS -name "*-weekly" -exec rm -rf '{}' ';'
+
+ perform_backups "-weekly"
+
+ exit 0;
+fi
+
+# DAILY BACKUPS
+
+# Delete daily backups 7 days old or more
+find $BACKUP_DIR -maxdepth 1 -mtime +$DAYS_TO_KEEP -name "*-daily" -exec rm -rf '{}' ';'
+
+perform_backups "-daily"
commit c0e15cd5d05a9b99c1a014f6437fe1914685a9a3
Author: rajanski <rajanski at users.noreply.github.com>
Date: Thu Jan 1 18:41:38 2015 +0100
Create pg_backup.sh
diff --git a/pg_backup.sh b/pg_backup.sh
new file mode 100644
index 0000000..4092cd0
--- /dev/null
+++ b/pg_backup.sh
@@ -0,0 +1,139 @@
+#!/bin/bash
+
+###########################
+####### LOAD CONFIG #######
+###########################
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ -c)
+ if [ -r "$2" ]; then
+ source "$2"
+ shift 2
+ else
+ ${ECHO} "Unreadable config file \"$2\"" 1>&2
+ exit 1
+ fi
+ ;;
+ *)
+ ${ECHO} "Unknown Option \"$1\"" 1>&2
+ exit 2
+ ;;
+ esac
+done
+
+if [ $# = 0 ]; then
+ SCRIPTPATH=$(cd ${0%/*} && pwd -P)
+ source $SCRIPTPATH/pg_backup.config
+fi;
+
+###########################
+#### PRE-BACKUP CHECKS ####
+###########################
+
+# Make sure we're running as the required backup user
+if [ "$BACKUP_USER" != "" -a "$(id -un)" != "$BACKUP_USER" ]; then
+ echo "This script must be run as $BACKUP_USER. Exiting." 1>&2
+ exit 1;
+fi;
+
+
+###########################
+### INITIALISE DEFAULTS ###
+###########################
+
+if [ ! $HOSTNAME ]; then
+ HOSTNAME="localhost"
+fi;
+
+if [ ! $USERNAME ]; then
+ USERNAME="postgres"
+fi;
+
+
+###########################
+#### START THE BACKUPS ####
+###########################
+
+
+FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y-\%m-\%d`/"
+
+echo "Making backup directory in $FINAL_BACKUP_DIR"
+
+if ! mkdir -p $FINAL_BACKUP_DIR; then
+ echo "Cannot create backup directory in $FINAL_BACKUP_DIR. Go and fix it!" 1>&2
+ exit 1;
+fi;
+
+
+###########################
+### SCHEMA-ONLY BACKUPS ###
+###########################
+
+for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
+do
+ SCHEMA_ONLY_CLAUSE="$SCHEMA_ONLY_CLAUSE or datname ~ '$SCHEMA_ONLY_DB'"
+done
+
+SCHEMA_ONLY_QUERY="select datname from pg_database where false $SCHEMA_ONLY_CLAUSE order by datname;"
+
+echo -e "\n\nPerforming schema-only backups"
+echo -e "--------------------------------------------\n"
+
+SCHEMA_ONLY_DB_LIST=`psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$SCHEMA_ONLY_QUERY" postgres`
+
+echo -e "The following databases were matched for schema-only backup:\n${SCHEMA_ONLY_DB_LIST}\n"
+
+for DATABASE in $SCHEMA_ONLY_DB_LIST
+do
+ echo "Schema-only backup of $DATABASE"
+
+ if ! pg_dump -Fp -s -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress; then
+ echo "[!!ERROR!!] Failed to backup database schema of $DATABASE" 1>&2
+ else
+ mv $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz
+ fi
+done
+
+
+###########################
+###### FULL BACKUPS #######
+###########################
+
+for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
+do
+ EXCLUDE_SCHEMA_ONLY_CLAUSE="$EXCLUDE_SCHEMA_ONLY_CLAUSE and datname !~ '$SCHEMA_ONLY_DB'"
+done
+
+FULL_BACKUP_QUERY="select datname from pg_database where not datistemplate and datallowconn $EXCLUDE_SCHEMA_ONLY_CLAUSE order by datname;"
+
+echo -e "\n\nPerforming full backups"
+echo -e "--------------------------------------------\n"
+
+for DATABASE in `psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$FULL_BACKUP_QUERY" postgres`
+do
+ if [ $ENABLE_PLAIN_BACKUPS = "yes" ]
+ then
+ echo "Plain backup of $DATABASE"
+
+ if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
+ echo "[!!ERROR!!] Failed to produce plain backup database $DATABASE" 1>&2
+ else
+ mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
+ fi
+ fi
+
+ if [ $ENABLE_CUSTOM_BACKUPS = "yes" ]
+ then
+ echo "Custom backup of $DATABASE"
+
+ if ! pg_dump -Fc -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" -f $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress; then
+ echo "[!!ERROR!!] Failed to produce custom backup database $DATABASE" 1>&2
+ else
+ mv $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress $FINAL_BACKUP_DIR"$DATABASE".custom
+ fi
+ fi
+
+done
+
+echo -e "\nAll database backups complete!"
commit 68692261617167a130bddc62b5b3368f97e6fddc
Author: rajanski <rajanski at users.noreply.github.com>
Date: Thu Jan 1 18:40:38 2015 +0100
Create pg_backup.config
diff --git a/pg_backup.config b/pg_backup.config
new file mode 100644
index 0000000..4781280
--- /dev/null
+++ b/pg_backup.config
@@ -0,0 +1,42 @@
+#############################
+## POSTGRESQL BACKUP CONFIG ##
+##############################
+
+# Optional system user to run backups as. If the user the script is running as doesn't match this
+# the script terminates. Leave blank to skip check.
+BACKUP_USER=
+
+# Optional hostname to adhere to pg_hba policies. Will default to "localhost" if none specified.
+HOSTNAME=
+
+# Optional username to connect to database as. Will default to "postgres" if none specified.
+USERNAME=
+
+# This dir will be created if it doesn't exist. This must be writable by the user the script is
+# running as.
+BACKUP_DIR=/home/backups/database/postgresql/
+
+# List of strings to match against in database name, separated by space or comma, for which we only
+# wish to keep a backup of the schema, not the data. Any database names which contain any of these
+# values will be considered candidates. (e.g. "system_log" will match "dev_system_log_2010-01")
+SCHEMA_ONLY_LIST=""
+
+# Will produce a custom-format backup if set to "yes"
+ENABLE_CUSTOM_BACKUPS=yes
+
+# Will produce a gzipped plain-format backup if set to "yes"
+ENABLE_PLAIN_BACKUPS=yes
+
+
+#### SETTINGS FOR ROTATED BACKUPS ####
+
+# Which day to take the weekly backup from (1-7 = Monday-Sunday)
+DAY_OF_WEEK_TO_KEEP=5
+
+# Number of days to keep daily backups
+DAYS_TO_KEEP=7
+
+# How many weeks to keep weekly backups
+WEEKS_TO_KEEP=5
+
+######################################
commit 86dd98be1eff3bd736c7bbd4f9193e0dec19421d
Author: rajanski <rajanski at users.noreply.github.com>
Date: Thu Jan 1 18:39:37 2015 +0100
Update README.md
diff --git a/README.md b/README.md
index 8e4b8fa..0ae6eb9 100644
--- a/README.md
+++ b/README.md
@@ -2,3 +2,7 @@ postgresql_auto_backup_s3
=========================
Backup Scripts to automate postgresql backups and push to s3 via stdin, plus rotation logic
+
+Original code from https://wiki.postgresql.org/wiki/Automated_Backup_on_Linux.
+
+Will be adapted to wrk with s3cmd stdin pushing
commit 3708cd554d9442c496925a2b76877eec20731d4f
Author: rajanski <rajanski at users.noreply.github.com>
Date: Thu Jan 1 18:38:03 2015 +0100
Initial commit
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..8e4b8fa
--- /dev/null
+++ b/README.md
@@ -0,0 +1,4 @@
+postgresql_auto_backup_s3
+=========================
+
+Backup Scripts to automate postgresql backups and push to s3 via stdin, plus rotation logic
-----------------------------------------------------------------------
hooks/post-receive
--
postgresql_auto_backup_s3
More information about the Bps-public-commit
mailing list