[Bps-public-commit] postgresql_auto_backup_s3 branch update-for-hostedrt-backups created. 6a7521f293e260aed77bc969d389d1a20b79c302
BPS Git Server
git at git.bestpractical.com
Tue Mar 15 00:22:27 UTC 2022
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "postgresql_auto_backup_s3".
The branch, update-for-hostedrt-backups has been created
at 6a7521f293e260aed77bc969d389d1a20b79c302 (commit)
- Log -----------------------------------------------------------------
commit 6a7521f293e260aed77bc969d389d1a20b79c302
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Mon Mar 14 19:19:08 2022 -0500
Remove commented out backup dir code
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index acbb613..d66c360 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -63,14 +63,6 @@ function perform_backups()
SUFFIX=$1
FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y\%m\%d`$SUFFIX-"
- #echo "Making backup directory in $FINAL_BACKUP_DIR"
-
- #if ! mkdir -p $FINAL_BACKUP_DIR; then
- # echo "Cannot create backup directory in $FINAL_BACKUP_DIR. Go and fix it!" 1>&2
- # exit 1;
- #fi;
-
-
###########################
###### FULL BACKUPS #######
###########################
commit d29a3684c06bb3ca9cc7736bbf56aea1e42819ac
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Mon Mar 14 19:07:43 2022 -0500
Update s3cmd commands to aws-cli
aws-cli is the recommended tool from AWS and supports input from
STDIN for s3 cp commands.
This commit also updates the backup rotation logic which was not
completed in the original version.
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index f0f2421..acbb613 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -82,10 +82,10 @@ function perform_backups()
do
echo "$DATABASE"
- if ! ( pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --table=sessions --schema-only; pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --exclude-table=sessions ) | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
+ if ! ( pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --table=sessions --schema-only; pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --exclude-table=sessions ) | gzip --stdout | aws s3 cp - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
echo "[!!ERROR!!] Failed to produce backup of database $DATABASE" 1>&2
else
- s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
+ aws s3 mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
fi
done
@@ -100,7 +100,9 @@ DAY_OF_MONTH=`date +%d`
if [[ "$DAY_OF_MONTH" -eq "1" ]];
then
# Delete all expired monthly directories
- find $BACKUP_DIR -maxdepth 1 -name "*-monthly" -exec rm -rf '{}' ';'
+ for backup in $(aws s3 ls $BACKUP_DIR | awk '{print $4}' | grep '\-monthly'); do
+ aws s3 rm $BACKUP_DIR"$backup"
+ done
perform_backups "-monthly"
@@ -110,13 +112,19 @@ fi
# WEEKLY BACKUPS
DAY_OF_WEEK=`date +%u` #1-7 (Monday-Sunday)
-EXPIRED_DAYS=`expr $((($WEEKS_TO_KEEP * 7) + 1))`
if [[ "$DAY_OF_WEEK" -eq "$DAY_OF_WEEK_TO_KEEP" ]];
then
- # Delete all expired weekly directories
- find $BACKUP_DIR -maxdepth 1 -mtime +$EXPIRED_DAYS -name "*-weekly" -exec rm -rf '{}' ';'
-
+ weekly_deldate=`date +"%Y%m%d" --date="$WEEKS_TO_KEEP days ago"`
+
+ for backup in $(aws s3 ls $BACKUP_DIR | awk '{print $4}' | grep '\-weekly'); do
+ weekly_backup_date=${backup:0:8}
+
+ if [[ "$weekly_backup_date" -le "$weekly_deldate" ]]; then
+ aws s3 rm $BACKUP_DIR"$backup"
+ fi
+ done
+
perform_backups "-weekly"
exit 0;
@@ -124,16 +132,14 @@ fi
# DAILY BACKUPS
-# Delete daily backups $DAYS_TO_KEEP days old or more
-#find $BACKUP_DIR -maxdepth 1 -mtime +$DAYS_TO_KEEP -name "*-daily" -exec rm -rf '{}' ';'
+daily_deldate=`date +"%Y%m%d" --date="$DAYS_TO_KEEP days ago"`
-#set $deldate to date minus $DAYS_TO_KEEP
-deldate=`date +"%Y%m%d" --date="$DAYS_TO_KEEP days ago"`
+for backup in $(aws s3 ls $BACKUP_DIR | awk '{print $4}' | grep '\-daily'); do
+ daily_backup_date=${backup:0:8}
-#remove leading bucket name from s3 object displayed by "s3cmd ls $bucketname"
-#compare first 8 chars of s3 object name (minus bucket name) with $deldate,
-#if first 8 chars of s3 object name::int <= $deldate then delete those s3 objects
-s3cmd ls $BACKUP_DIR | awk -v bucket="$BACKUP_DIR" -v deldate="$deldate" '{gsub(bucket,"",$4);if (substr($4,0,9)<=deldate) system("s3cmd info " bucket$4)}'
-#replace s3cmd info by s3cmd del
+ if [[ "$daily_backup_date" -le "$daily_deldate" ]]; then
+ aws s3 rm $BACKUP_DIR"$backup"
+ fi
+done
perform_backups "-daily"
commit f9a2f7dfc2a264b6ba12541adeb39e56cb808c21
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 15:27:27 2022 -0600
Fix -c option to load config
The -c option to load a non-default config file didn't work for
pg_backup.sh. This commit copies the logic from
pg_backup_rotated.sh over to pg_backup.sh, which does work.
diff --git a/pg_backup.sh b/pg_backup.sh
index 93acb14..a44e3b7 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -3,17 +3,12 @@
###########################
####### LOAD CONFIG #######
###########################
-
+
while [[ "$#" -gt "0" ]]; do
case $1 in
-c)
- if [[ -r "$2" ]]; then
- source "$2"
- shift 2
- else
- ${ECHO} "Unreadable config file \"$2\"" 1>&2
- exit 1
- fi
+ CONFIG_FILE_PATH="$2"
+ shift 2
;;
*)
${ECHO} "Unknown Option \"$1\"" 1>&2
@@ -21,13 +16,21 @@ while [[ "$#" -gt "0" ]]; do
;;
esac
done
-
-if [[ "$#" -eq "0" ]]; then
+
+if [[ -z "$CONFIG_FILE_PATH" ]]; then
SCRIPT=`realpath $0`
SCRIPTPATH=`dirname $SCRIPT`
- source $SCRIPTPATH/pg_backup.config
-fi;
-
+ CONFIG_FILE_PATH="${SCRIPTPATH}/pg_backup.config"
+fi
+
+if [[ ! -r "${CONFIG_FILE_PATH}" ]]; then
+ echo "Could not load config file from ${CONFIG_FILE_PATH}" 1>&2
+ exit 1
+fi
+
+source "${CONFIG_FILE_PATH}"
+
+
###########################
#### PRE-BACKUP CHECKS ####
###########################
commit 5686cdb025b0f0f6de99ae67336312defcfcca35
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 15:19:24 2022 -0600
Convert tabs to spaces (retab)
diff --git a/pg_backup.sh b/pg_backup.sh
index 4e7817b..93acb14 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -34,8 +34,8 @@ fi;
# Make sure we're running as the required backup user
if [[ "$BACKUP_USER" != "" && "$(id -un)" != "$BACKUP_USER" ]]; then
- echo "This script must be run as $BACKUP_USER. Exiting." 1>&2
- exit 1;
+ echo "This script must be run as $BACKUP_USER. Exiting." 1>&2
+ exit 1;
fi;
@@ -44,11 +44,11 @@ fi;
###########################
if [[ -z "$HOSTNAME" ]]; then
- HOSTNAME="localhost"
+ HOSTNAME="localhost"
fi;
if [[ -z "$USERNAME" ]]; then
- USERNAME="postgres"
+ USERNAME="postgres"
fi;
@@ -62,8 +62,8 @@ FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y-\%m-\%d`/"
echo "Making backup directory in $FINAL_BACKUP_DIR"
if ! mkdir -p $FINAL_BACKUP_DIR; then
- echo "Cannot create backup directory in $FINAL_BACKUP_DIR. Go and fix it!" 1>&2
- exit 1;
+ echo "Cannot create backup directory in $FINAL_BACKUP_DIR. Go and fix it!" 1>&2
+ exit 1;
fi;
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index 675ca7b..f0f2421 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -36,8 +36,8 @@ source "${CONFIG_FILE_PATH}"
# Make sure we're running as the required backup user
if [[ "$BACKUP_USER" != "" && "$(id -un)" != "$BACKUP_USER" ]]; then
- echo "This script must be run as $BACKUP_USER. Exiting." 1>&2
- exit 1
+ echo "This script must be run as $BACKUP_USER. Exiting." 1>&2
+ exit 1
fi
@@ -46,11 +46,11 @@ fi
###########################
if [[ -z "$HOSTNAME" ]]; then
- HOSTNAME="localhost"
+ HOSTNAME="localhost"
fi;
if [[ -z "$USERNAME" ]]; then
- USERNAME="postgres"
+ USERNAME="postgres"
fi;
@@ -60,37 +60,37 @@ fi;
function perform_backups()
{
- SUFFIX=$1
- FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y\%m\%d`$SUFFIX-"
+ SUFFIX=$1
+ FINAL_BACKUP_DIR=$BACKUP_DIR"`date +\%Y\%m\%d`$SUFFIX-"
- #echo "Making backup directory in $FINAL_BACKUP_DIR"
+ #echo "Making backup directory in $FINAL_BACKUP_DIR"
- #if ! mkdir -p $FINAL_BACKUP_DIR; then
- # echo "Cannot create backup directory in $FINAL_BACKUP_DIR. Go and fix it!" 1>&2
- # exit 1;
- #fi;
+ #if ! mkdir -p $FINAL_BACKUP_DIR; then
+ # echo "Cannot create backup directory in $FINAL_BACKUP_DIR. Go and fix it!" 1>&2
+ # exit 1;
+ #fi;
- ###########################
- ###### FULL BACKUPS #######
- ###########################
+ ###########################
+ ###### FULL BACKUPS #######
+ ###########################
- echo -e "\n\nPerforming full backups"
- echo -e "--------------------------------------------\n"
+ echo -e "\n\nPerforming full backups"
+ echo -e "--------------------------------------------\n"
- for DATABASE in ${BACKUP_DB_LIST//,/ }
- do
- echo "$DATABASE"
+ for DATABASE in ${BACKUP_DB_LIST//,/ }
+ do
+ echo "$DATABASE"
- if ! ( pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --table=sessions --schema-only; pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --exclude-table=sessions ) | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
- echo "[!!ERROR!!] Failed to produce backup of database $DATABASE" 1>&2
- else
- s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
- fi
+ if ! ( pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --table=sessions --schema-only; pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --exclude-table=sessions ) | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
+ echo "[!!ERROR!!] Failed to produce backup of database $DATABASE" 1>&2
+ else
+ s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
+ fi
- done
+ done
- echo -e "\nAll database backups complete!"
+ echo -e "\nAll database backups complete!"
}
# MONTHLY BACKUPS
@@ -99,12 +99,12 @@ DAY_OF_MONTH=`date +%d`
if [[ "$DAY_OF_MONTH" -eq "1" ]];
then
- # Delete all expired monthly directories
- find $BACKUP_DIR -maxdepth 1 -name "*-monthly" -exec rm -rf '{}' ';'
+ # Delete all expired monthly directories
+ find $BACKUP_DIR -maxdepth 1 -name "*-monthly" -exec rm -rf '{}' ';'
- perform_backups "-monthly"
+ perform_backups "-monthly"
- exit 0;
+ exit 0;
fi
# WEEKLY BACKUPS
@@ -114,12 +114,12 @@ EXPIRED_DAYS=`expr $((($WEEKS_TO_KEEP * 7) + 1))`
if [[ "$DAY_OF_WEEK" -eq "$DAY_OF_WEEK_TO_KEEP" ]];
then
- # Delete all expired weekly directories
- find $BACKUP_DIR -maxdepth 1 -mtime +$EXPIRED_DAYS -name "*-weekly" -exec rm -rf '{}' ';'
+ # Delete all expired weekly directories
+ find $BACKUP_DIR -maxdepth 1 -mtime +$EXPIRED_DAYS -name "*-weekly" -exec rm -rf '{}' ';'
- perform_backups "-weekly"
+ perform_backups "-weekly"
- exit 0;
+ exit 0;
fi
# DAILY BACKUPS
commit 6e5258167dbde23458e09010f1be4c1526f93d02
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 15:03:01 2022 -0600
Update pg_dump commands for RT specific backups
The pg_dump commands now specifically exclude the sessions table
data as outlined in our documentation.
diff --git a/pg_backup.sh b/pg_backup.sh
index 9bfd8fe..4e7817b 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -78,7 +78,7 @@ for DATABASE in ${BACKUP_DB_LIST//,/ }
do
echo "$DATABASE"
- if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
+ if ! ( pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --table=sessions --schema-only; pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --exclude-table=sessions ) | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
echo "[!!ERROR!!] Failed to produce backup of database $DATABASE" 1>&2
else
mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index cf470d1..675ca7b 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -82,7 +82,7 @@ function perform_backups()
do
echo "$DATABASE"
- if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
+ if ! ( pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --table=sessions --schema-only; pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --exclude-table=sessions ) | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
echo "[!!ERROR!!] Failed to produce backup of database $DATABASE" 1>&2
else
s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
commit 6dd34bd76e8d646dbca92104941274b47f9ab018
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 14:39:52 2022 -0600
Remove custom and plain backup configuration
This commit removes the option and command to create custom
backups. The scripts now only create plain backups and don't have
a configurable option to enable or disable the backup creation.
diff --git a/pg_backup.config b/pg_backup.config
index c84a5c5..8374983 100644
--- a/pg_backup.config
+++ b/pg_backup.config
@@ -21,12 +21,6 @@ BACKUP_DIR=s3://pg_backups/
# List of database names to backup, separated by space or comma.
BACKUP_DB_LIST=""
-# Will produce a custom-format backup if set to "yes"
-ENABLE_CUSTOM_BACKUPS=yes
-
-# Will produce a gzipped plain-format backup if set to "yes"
-ENABLE_PLAIN_BACKUPS=yes
-
#### SETTINGS FOR ROTATED BACKUPS ####
diff --git a/pg_backup.sh b/pg_backup.sh
index ea2cecb..9bfd8fe 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -76,27 +76,13 @@ echo -e "--------------------------------------------\n"
for DATABASE in ${BACKUP_DB_LIST//,/ }
do
- if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
- then
- echo "Plain backup of $DATABASE"
-
- if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
- echo "[!!ERROR!!] Failed to produce plain backup database $DATABASE" 1>&2
- else
- mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
- fi
- fi
-
- if [[ $ENABLE_CUSTOM_BACKUPS == "yes" ]]
- then
- echo "Custom backup of $DATABASE"
-
- if ! pg_dump -Fc -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" -f $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress; then
- echo "[!!ERROR!!] Failed to produce custom backup database $DATABASE" 1>&2
- else
- mv $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress $FINAL_BACKUP_DIR"$DATABASE".custom
- fi
- fi
+ echo "$DATABASE"
+
+ if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
+ echo "[!!ERROR!!] Failed to produce backup of database $DATABASE" 1>&2
+ else
+ mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
+ fi
done
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index b6b6105..cf470d1 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -80,26 +80,13 @@ function perform_backups()
for DATABASE in ${BACKUP_DB_LIST//,/ }
do
- if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
- then
- echo "Plain backup of $DATABASE"
-
- if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
- echo "[!!ERROR!!] Failed to produce plain backup database schema of $DATABASE" 1>&2
- else
- s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
- fi
- fi
-
- if [[ "$ENABLE_CUSTOM_BACKUPS" == "yes" ]]
- then
- echo "Custom backup of $DATABASE"
- if ! pg_dump -Fc -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress --no-encrypt; then
- echo "[!!ERROR!!] Failed to produce custom backup database schema of $DATABASE" 1>&2
- else
- s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress $FINAL_BACKUP_DIR"$DATABASE".custom
- fi
- fi
+ echo "$DATABASE"
+
+ if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
+ echo "[!!ERROR!!] Failed to produce backup of database $DATABASE" 1>&2
+ else
+ s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
+ fi
done
commit c43d8d091fefcaeee278358379c55605dae5e799
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 14:16:07 2022 -0600
Remove schema only backups and db exclusion match
This commit removes the schema only backup functionality, as well as
simplifying functionality to only backup the explicitly defined
databases.
diff --git a/pg_backup.config b/pg_backup.config
index 5e97702..c84a5c5 100644
--- a/pg_backup.config
+++ b/pg_backup.config
@@ -18,10 +18,8 @@ USERNAME=
#change to s3 bucket
BACKUP_DIR=s3://pg_backups/
-# List of strings to match against in database name, separated by space or comma, for which we only
-# wish to keep a backup of the schema, not the data. Any database names which contain any of these
-# values will be considered candidates. (e.g. "system_log" will match "dev_system_log_2010-01")
-SCHEMA_ONLY_LIST=""
+# List of database names to backup, separated by space or comma.
+BACKUP_DB_LIST=""
# Will produce a custom-format backup if set to "yes"
ENABLE_CUSTOM_BACKUPS=yes
diff --git a/pg_backup.sh b/pg_backup.sh
index 320398a..ea2cecb 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -67,51 +67,14 @@ if ! mkdir -p $FINAL_BACKUP_DIR; then
fi;
-###########################
-### SCHEMA-ONLY BACKUPS ###
-###########################
-
-for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
-do
- SCHEMA_ONLY_CLAUSE="$SCHEMA_ONLY_CLAUSE or datname ~ '$SCHEMA_ONLY_DB'"
-done
-
-SCHEMA_ONLY_QUERY="select datname from pg_database where false $SCHEMA_ONLY_CLAUSE order by datname;"
-
-echo -e "\n\nPerforming schema-only backups"
-echo -e "--------------------------------------------\n"
-
-SCHEMA_ONLY_DB_LIST=`psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$SCHEMA_ONLY_QUERY" postgres`
-
-echo -e "The following databases were matched for schema-only backup:\n${SCHEMA_ONLY_DB_LIST}\n"
-
-for DATABASE in $SCHEMA_ONLY_DB_LIST
-do
- echo "Schema-only backup of $DATABASE"
-
- if ! pg_dump -Fp -s -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress; then
- echo "[!!ERROR!!] Failed to backup database schema of $DATABASE" 1>&2
- else
- mv $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz
- fi
-done
-
-
###########################
###### FULL BACKUPS #######
###########################
-for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
-do
- EXCLUDE_SCHEMA_ONLY_CLAUSE="$EXCLUDE_SCHEMA_ONLY_CLAUSE and datname !~ '$SCHEMA_ONLY_DB'"
-done
-
-FULL_BACKUP_QUERY="select datname from pg_database where not datistemplate and datallowconn $EXCLUDE_SCHEMA_ONLY_CLAUSE order by datname;"
-
echo -e "\n\nPerforming full backups"
echo -e "--------------------------------------------\n"
-for DATABASE in `psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$FULL_BACKUP_QUERY" postgres`
+for DATABASE in ${BACKUP_DB_LIST//,/ }
do
if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
then
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index 8d71796..b6b6105 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -71,51 +71,14 @@ function perform_backups()
#fi;
- ###########################
- ### SCHEMA-ONLY BACKUPS ###
- ###########################
-
- for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
- do
- SCHEMA_ONLY_CLAUSE="$SCHEMA_ONLY_CLAUSE or datname ~ '$SCHEMA_ONLY_DB'"
- done
-
- SCHEMA_ONLY_QUERY="select datname from pg_database where false $SCHEMA_ONLY_CLAUSE order by datname;"
-
- echo -e "\n\nPerforming schema-only backups"
- echo -e "--------------------------------------------\n"
-
- SCHEMA_ONLY_DB_LIST=`psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$SCHEMA_ONLY_QUERY" postgres`
-
- echo -e "The following databases were matched for schema-only backup:\n${SCHEMA_ONLY_DB_LIST}\n"
-
- for DATABASE in $SCHEMA_ONLY_DB_LIST
- do
- echo "Schema-only backup of $DATABASE"
-
- if ! pg_dump -Fp -s -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress --no-encrypt; then
- echo "[!!ERROR!!] Failed to backup database schema of $DATABASE" 1>&2
- else
- s3cmd mv $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz
- fi
- done
-
-
###########################
###### FULL BACKUPS #######
###########################
- for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
- do
- EXCLUDE_SCHEMA_ONLY_CLAUSE="$EXCLUDE_SCHEMA_ONLY_CLAUSE and datname !~ '$SCHEMA_ONLY_DB'"
- done
-
- FULL_BACKUP_QUERY="select datname from pg_database where not datistemplate and datallowconn $EXCLUDE_SCHEMA_ONLY_CLAUSE order by datname;"
-
echo -e "\n\nPerforming full backups"
echo -e "--------------------------------------------\n"
- for DATABASE in `psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$FULL_BACKUP_QUERY" postgres`
+ for DATABASE in ${BACKUP_DB_LIST//,/ }
do
if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
then
commit fc022712767caa3929b2dff7ba67c0571e695b65
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 12:00:01 2022 -0600
Fix config file path generation
diff --git a/pg_backup.sh b/pg_backup.sh
index a35a3a1..320398a 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -23,7 +23,8 @@ while [[ "$#" -gt "0" ]]; do
done
if [[ "$#" -eq "0" ]]; then
- SCRIPTPATH=$(cd ${0%/*} && pwd -P)
+ SCRIPT=`realpath $0`
+ SCRIPTPATH=`dirname $SCRIPT`
source $SCRIPTPATH/pg_backup.config
fi;
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index 7380a5e..8d71796 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -18,7 +18,8 @@ while [[ "$#" -gt "0" ]]; do
done
if [[ -z "$CONFIG_FILE_PATH" ]]; then
- SCRIPTPATH=$(cd ${0%/*} && pwd -P)
+ SCRIPT=`realpath $0`
+ SCRIPTPATH=`dirname $SCRIPT`
CONFIG_FILE_PATH="${SCRIPTPATH}/pg_backup.config"
fi
commit 1b6052e3854191a48b6c27b730fd717e44e0c2e3
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 11:36:50 2022 -0600
Update comparison operators
The scripts within this project mix POSIX compliant syntax with
modern bash syntax. Rather than mixing usage, since the scripts
are defined specifically to run /bin/bash, this commit updates the
test and comparison operators to modern bash syntax.
- update "-a" booleen to &&
"-a" is deprecated and its use is discouraged.
- update single bracket "[]" test to double bracket "[[]]"
mixing single and double brackets within a codebase isn't
consistent or considered best practice.
- update "=" to "=="
mixing single and double equal is inconsistent.
- update variable quoting in comparison
for correct equality comparison and interpolation best practice.
diff --git a/pg_backup.sh b/pg_backup.sh
index 4092cd0..a35a3a1 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -4,10 +4,10 @@
####### LOAD CONFIG #######
###########################
-while [ $# -gt 0 ]; do
+while [[ "$#" -gt "0" ]]; do
case $1 in
-c)
- if [ -r "$2" ]; then
+ if [[ -r "$2" ]]; then
source "$2"
shift 2
else
@@ -22,7 +22,7 @@ while [ $# -gt 0 ]; do
esac
done
-if [ $# = 0 ]; then
+if [[ "$#" -eq "0" ]]; then
SCRIPTPATH=$(cd ${0%/*} && pwd -P)
source $SCRIPTPATH/pg_backup.config
fi;
@@ -32,7 +32,7 @@ fi;
###########################
# Make sure we're running as the required backup user
-if [ "$BACKUP_USER" != "" -a "$(id -un)" != "$BACKUP_USER" ]; then
+if [[ "$BACKUP_USER" != "" && "$(id -un)" != "$BACKUP_USER" ]]; then
echo "This script must be run as $BACKUP_USER. Exiting." 1>&2
exit 1;
fi;
@@ -42,11 +42,11 @@ fi;
### INITIALISE DEFAULTS ###
###########################
-if [ ! $HOSTNAME ]; then
+if [[ -z "$HOSTNAME" ]]; then
HOSTNAME="localhost"
fi;
-if [ ! $USERNAME ]; then
+if [[ -z "$USERNAME" ]]; then
USERNAME="postgres"
fi;
@@ -112,7 +112,7 @@ echo -e "--------------------------------------------\n"
for DATABASE in `psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$FULL_BACKUP_QUERY" postgres`
do
- if [ $ENABLE_PLAIN_BACKUPS = "yes" ]
+ if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
then
echo "Plain backup of $DATABASE"
@@ -123,7 +123,7 @@ do
fi
fi
- if [ $ENABLE_CUSTOM_BACKUPS = "yes" ]
+ if [[ $ENABLE_CUSTOM_BACKUPS == "yes" ]]
then
echo "Custom backup of $DATABASE"
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index 10e36d6..7380a5e 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -4,7 +4,7 @@
####### LOAD CONFIG #######
###########################
-while [ $# -gt 0 ]; do
+while [[ "$#" -gt "0" ]]; do
case $1 in
-c)
CONFIG_FILE_PATH="$2"
@@ -17,12 +17,12 @@ while [ $# -gt 0 ]; do
esac
done
-if [ -z $CONFIG_FILE_PATH ] ; then
+if [[ -z "$CONFIG_FILE_PATH" ]]; then
SCRIPTPATH=$(cd ${0%/*} && pwd -P)
CONFIG_FILE_PATH="${SCRIPTPATH}/pg_backup.config"
fi
-if [ ! -r ${CONFIG_FILE_PATH} ] ; then
+if [[ ! -r "${CONFIG_FILE_PATH}" ]]; then
echo "Could not load config file from ${CONFIG_FILE_PATH}" 1>&2
exit 1
fi
@@ -34,7 +34,7 @@ source "${CONFIG_FILE_PATH}"
###########################
# Make sure we're running as the required backup user
-if [ "$BACKUP_USER" != "" -a "$(id -un)" != "$BACKUP_USER" ] ; then
+if [[ "$BACKUP_USER" != "" && "$(id -un)" != "$BACKUP_USER" ]]; then
echo "This script must be run as $BACKUP_USER. Exiting." 1>&2
exit 1
fi
@@ -44,11 +44,11 @@ fi
### INITIALISE DEFAULTS ###
###########################
-if [ ! $HOSTNAME ]; then
+if [[ -z "$HOSTNAME" ]]; then
HOSTNAME="localhost"
fi;
-if [ ! $USERNAME ]; then
+if [[ -z "$USERNAME" ]]; then
USERNAME="postgres"
fi;
@@ -116,7 +116,7 @@ function perform_backups()
for DATABASE in `psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$FULL_BACKUP_QUERY" postgres`
do
- if [ $ENABLE_PLAIN_BACKUPS = "yes" ]
+ if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
then
echo "Plain backup of $DATABASE"
@@ -127,7 +127,7 @@ function perform_backups()
fi
fi
- if [ $ENABLE_CUSTOM_BACKUPS = "yes" ]
+ if [[ "$ENABLE_CUSTOM_BACKUPS" == "yes" ]]
then
echo "Custom backup of $DATABASE"
if ! pg_dump -Fc -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress --no-encrypt; then
@@ -146,7 +146,7 @@ function perform_backups()
DAY_OF_MONTH=`date +%d`
-if [ $DAY_OF_MONTH -eq 1 ];
+if [[ "$DAY_OF_MONTH" -eq "1" ]];
then
# Delete all expired monthly directories
find $BACKUP_DIR -maxdepth 1 -name "*-monthly" -exec rm -rf '{}' ';'
@@ -161,7 +161,7 @@ fi
DAY_OF_WEEK=`date +%u` #1-7 (Monday-Sunday)
EXPIRED_DAYS=`expr $((($WEEKS_TO_KEEP * 7) + 1))`
-if [ $DAY_OF_WEEK = $DAY_OF_WEEK_TO_KEEP ];
+if [[ "$DAY_OF_WEEK" -eq "$DAY_OF_WEEK_TO_KEEP" ]];
then
# Delete all expired weekly directories
find $BACKUP_DIR -maxdepth 1 -mtime +$EXPIRED_DAYS -name "*-weekly" -exec rm -rf '{}' ';'
-----------------------------------------------------------------------
hooks/post-receive
--
postgresql_auto_backup_s3
More information about the Bps-public-commit
mailing list