[Bps-public-commit] postgresql_auto_backup_s3 branch update-for-hostedrt-backups created. 6e5258167dbde23458e09010f1be4c1526f93d02
BPS Git Server
git at git.bestpractical.com
Fri Mar 11 21:04:59 UTC 2022
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "postgresql_auto_backup_s3".
The branch, update-for-hostedrt-backups has been created
at 6e5258167dbde23458e09010f1be4c1526f93d02 (commit)
- Log -----------------------------------------------------------------
commit 6e5258167dbde23458e09010f1be4c1526f93d02
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 15:03:01 2022 -0600
Update pg_dump commands for RT specific backups
The pg_dump commands now specifically exclude the sessions table
data as outlined in our documentation.
diff --git a/pg_backup.sh b/pg_backup.sh
index 9bfd8fe..4e7817b 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -78,7 +78,7 @@ for DATABASE in ${BACKUP_DB_LIST//,/ }
do
echo "$DATABASE"
- if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
+ if ! ( pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --table=sessions --schema-only; pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --exclude-table=sessions ) | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
echo "[!!ERROR!!] Failed to produce backup of database $DATABASE" 1>&2
else
mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index cf470d1..675ca7b 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -82,7 +82,7 @@ function perform_backups()
do
echo "$DATABASE"
- if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
+ if ! ( pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --table=sessions --schema-only; pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" --exclude-table=sessions ) | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
echo "[!!ERROR!!] Failed to produce backup of database $DATABASE" 1>&2
else
s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
commit 6dd34bd76e8d646dbca92104941274b47f9ab018
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 14:39:52 2022 -0600
Remove custom and plain backup configuration
This commit removes the option and command to create custom
backups. The scripts now only create plain backups and don't have
a configurable option to enable or disable the backup creation.
diff --git a/pg_backup.config b/pg_backup.config
index c84a5c5..8374983 100644
--- a/pg_backup.config
+++ b/pg_backup.config
@@ -21,12 +21,6 @@ BACKUP_DIR=s3://pg_backups/
# List of database names to backup, separated by space or comma.
BACKUP_DB_LIST=""
-# Will produce a custom-format backup if set to "yes"
-ENABLE_CUSTOM_BACKUPS=yes
-
-# Will produce a gzipped plain-format backup if set to "yes"
-ENABLE_PLAIN_BACKUPS=yes
-
#### SETTINGS FOR ROTATED BACKUPS ####
diff --git a/pg_backup.sh b/pg_backup.sh
index ea2cecb..9bfd8fe 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -76,27 +76,13 @@ echo -e "--------------------------------------------\n"
for DATABASE in ${BACKUP_DB_LIST//,/ }
do
- if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
- then
- echo "Plain backup of $DATABASE"
-
- if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
- echo "[!!ERROR!!] Failed to produce plain backup database $DATABASE" 1>&2
- else
- mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
- fi
- fi
-
- if [[ $ENABLE_CUSTOM_BACKUPS == "yes" ]]
- then
- echo "Custom backup of $DATABASE"
-
- if ! pg_dump -Fc -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" -f $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress; then
- echo "[!!ERROR!!] Failed to produce custom backup database $DATABASE" 1>&2
- else
- mv $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress $FINAL_BACKUP_DIR"$DATABASE".custom
- fi
- fi
+ echo "$DATABASE"
+
+ if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress; then
+ echo "[!!ERROR!!] Failed to produce backup of database $DATABASE" 1>&2
+ else
+ mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
+ fi
done
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index b6b6105..cf470d1 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -80,26 +80,13 @@ function perform_backups()
for DATABASE in ${BACKUP_DB_LIST//,/ }
do
- if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
- then
- echo "Plain backup of $DATABASE"
-
- if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
- echo "[!!ERROR!!] Failed to produce plain backup database schema of $DATABASE" 1>&2
- else
- s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
- fi
- fi
-
- if [[ "$ENABLE_CUSTOM_BACKUPS" == "yes" ]]
- then
- echo "Custom backup of $DATABASE"
- if ! pg_dump -Fc -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress --no-encrypt; then
- echo "[!!ERROR!!] Failed to produce custom backup database schema of $DATABASE" 1>&2
- else
- s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress $FINAL_BACKUP_DIR"$DATABASE".custom
- fi
- fi
+ echo "$DATABASE"
+
+ if ! pg_dump -Fp -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress --no-encrypt; then
+ echo "[!!ERROR!!] Failed to produce backup of database $DATABASE" 1>&2
+ else
+ s3cmd mv $FINAL_BACKUP_DIR"$DATABASE".sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE".sql.gz
+ fi
done
commit c43d8d091fefcaeee278358379c55605dae5e799
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 14:16:07 2022 -0600
Remove schema only backups and db exclusion match
This commit removes the schema only backup functionality, as well as
simplifying functionality to only backup the explicitly defined
databases.
diff --git a/pg_backup.config b/pg_backup.config
index 5e97702..c84a5c5 100644
--- a/pg_backup.config
+++ b/pg_backup.config
@@ -18,10 +18,8 @@ USERNAME=
#change to s3 bucket
BACKUP_DIR=s3://pg_backups/
-# List of strings to match against in database name, separated by space or comma, for which we only
-# wish to keep a backup of the schema, not the data. Any database names which contain any of these
-# values will be considered candidates. (e.g. "system_log" will match "dev_system_log_2010-01")
-SCHEMA_ONLY_LIST=""
+# List of database names to backup, separated by space or comma.
+BACKUP_DB_LIST=""
# Will produce a custom-format backup if set to "yes"
ENABLE_CUSTOM_BACKUPS=yes
diff --git a/pg_backup.sh b/pg_backup.sh
index 320398a..ea2cecb 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -67,51 +67,14 @@ if ! mkdir -p $FINAL_BACKUP_DIR; then
fi;
-###########################
-### SCHEMA-ONLY BACKUPS ###
-###########################
-
-for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
-do
- SCHEMA_ONLY_CLAUSE="$SCHEMA_ONLY_CLAUSE or datname ~ '$SCHEMA_ONLY_DB'"
-done
-
-SCHEMA_ONLY_QUERY="select datname from pg_database where false $SCHEMA_ONLY_CLAUSE order by datname;"
-
-echo -e "\n\nPerforming schema-only backups"
-echo -e "--------------------------------------------\n"
-
-SCHEMA_ONLY_DB_LIST=`psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$SCHEMA_ONLY_QUERY" postgres`
-
-echo -e "The following databases were matched for schema-only backup:\n${SCHEMA_ONLY_DB_LIST}\n"
-
-for DATABASE in $SCHEMA_ONLY_DB_LIST
-do
- echo "Schema-only backup of $DATABASE"
-
- if ! pg_dump -Fp -s -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip > $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress; then
- echo "[!!ERROR!!] Failed to backup database schema of $DATABASE" 1>&2
- else
- mv $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz
- fi
-done
-
-
###########################
###### FULL BACKUPS #######
###########################
-for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
-do
- EXCLUDE_SCHEMA_ONLY_CLAUSE="$EXCLUDE_SCHEMA_ONLY_CLAUSE and datname !~ '$SCHEMA_ONLY_DB'"
-done
-
-FULL_BACKUP_QUERY="select datname from pg_database where not datistemplate and datallowconn $EXCLUDE_SCHEMA_ONLY_CLAUSE order by datname;"
-
echo -e "\n\nPerforming full backups"
echo -e "--------------------------------------------\n"
-for DATABASE in `psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$FULL_BACKUP_QUERY" postgres`
+for DATABASE in ${BACKUP_DB_LIST//,/ }
do
if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
then
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index 8d71796..b6b6105 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -71,51 +71,14 @@ function perform_backups()
#fi;
- ###########################
- ### SCHEMA-ONLY BACKUPS ###
- ###########################
-
- for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
- do
- SCHEMA_ONLY_CLAUSE="$SCHEMA_ONLY_CLAUSE or datname ~ '$SCHEMA_ONLY_DB'"
- done
-
- SCHEMA_ONLY_QUERY="select datname from pg_database where false $SCHEMA_ONLY_CLAUSE order by datname;"
-
- echo -e "\n\nPerforming schema-only backups"
- echo -e "--------------------------------------------\n"
-
- SCHEMA_ONLY_DB_LIST=`psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$SCHEMA_ONLY_QUERY" postgres`
-
- echo -e "The following databases were matched for schema-only backup:\n${SCHEMA_ONLY_DB_LIST}\n"
-
- for DATABASE in $SCHEMA_ONLY_DB_LIST
- do
- echo "Schema-only backup of $DATABASE"
-
- if ! pg_dump -Fp -s -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress --no-encrypt; then
- echo "[!!ERROR!!] Failed to backup database schema of $DATABASE" 1>&2
- else
- s3cmd mv $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz.in_progress $FINAL_BACKUP_DIR"$DATABASE"_SCHEMA.sql.gz
- fi
- done
-
-
###########################
###### FULL BACKUPS #######
###########################
- for SCHEMA_ONLY_DB in ${SCHEMA_ONLY_LIST//,/ }
- do
- EXCLUDE_SCHEMA_ONLY_CLAUSE="$EXCLUDE_SCHEMA_ONLY_CLAUSE and datname !~ '$SCHEMA_ONLY_DB'"
- done
-
- FULL_BACKUP_QUERY="select datname from pg_database where not datistemplate and datallowconn $EXCLUDE_SCHEMA_ONLY_CLAUSE order by datname;"
-
echo -e "\n\nPerforming full backups"
echo -e "--------------------------------------------\n"
- for DATABASE in `psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$FULL_BACKUP_QUERY" postgres`
+ for DATABASE in ${BACKUP_DB_LIST//,/ }
do
if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
then
commit fc022712767caa3929b2dff7ba67c0571e695b65
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 12:00:01 2022 -0600
Fix config file path generation
diff --git a/pg_backup.sh b/pg_backup.sh
index a35a3a1..320398a 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -23,7 +23,8 @@ while [[ "$#" -gt "0" ]]; do
done
if [[ "$#" -eq "0" ]]; then
- SCRIPTPATH=$(cd ${0%/*} && pwd -P)
+ SCRIPT=`realpath $0`
+ SCRIPTPATH=`dirname $SCRIPT`
source $SCRIPTPATH/pg_backup.config
fi;
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index 7380a5e..8d71796 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -18,7 +18,8 @@ while [[ "$#" -gt "0" ]]; do
done
if [[ -z "$CONFIG_FILE_PATH" ]]; then
- SCRIPTPATH=$(cd ${0%/*} && pwd -P)
+ SCRIPT=`realpath $0`
+ SCRIPTPATH=`dirname $SCRIPT`
CONFIG_FILE_PATH="${SCRIPTPATH}/pg_backup.config"
fi
commit 1b6052e3854191a48b6c27b730fd717e44e0c2e3
Author: Blaine Motsinger <blaine at bestpractical.com>
Date: Fri Mar 11 11:36:50 2022 -0600
Update comparison operators
The scripts within this project mix POSIX compliant syntax with
modern bash syntax. Rather than mixing usage, since the scripts
are defined specifically to run /bin/bash, this commit updates the
test and comparison operators to modern bash syntax.
- update "-a" booleen to &&
"-a" is deprecated and its use is discouraged.
- update single bracket "[]" test to double bracket "[[]]"
mixing single and double brackets within a codebase isn't
consistent or considered best practice.
- update "=" to "=="
mixing single and double equal is inconsistent.
- update variable quoting in comparison
for correct equality comparison and interpolation best practice.
diff --git a/pg_backup.sh b/pg_backup.sh
index 4092cd0..a35a3a1 100644
--- a/pg_backup.sh
+++ b/pg_backup.sh
@@ -4,10 +4,10 @@
####### LOAD CONFIG #######
###########################
-while [ $# -gt 0 ]; do
+while [[ "$#" -gt "0" ]]; do
case $1 in
-c)
- if [ -r "$2" ]; then
+ if [[ -r "$2" ]]; then
source "$2"
shift 2
else
@@ -22,7 +22,7 @@ while [ $# -gt 0 ]; do
esac
done
-if [ $# = 0 ]; then
+if [[ "$#" -eq "0" ]]; then
SCRIPTPATH=$(cd ${0%/*} && pwd -P)
source $SCRIPTPATH/pg_backup.config
fi;
@@ -32,7 +32,7 @@ fi;
###########################
# Make sure we're running as the required backup user
-if [ "$BACKUP_USER" != "" -a "$(id -un)" != "$BACKUP_USER" ]; then
+if [[ "$BACKUP_USER" != "" && "$(id -un)" != "$BACKUP_USER" ]]; then
echo "This script must be run as $BACKUP_USER. Exiting." 1>&2
exit 1;
fi;
@@ -42,11 +42,11 @@ fi;
### INITIALISE DEFAULTS ###
###########################
-if [ ! $HOSTNAME ]; then
+if [[ -z "$HOSTNAME" ]]; then
HOSTNAME="localhost"
fi;
-if [ ! $USERNAME ]; then
+if [[ -z "$USERNAME" ]]; then
USERNAME="postgres"
fi;
@@ -112,7 +112,7 @@ echo -e "--------------------------------------------\n"
for DATABASE in `psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$FULL_BACKUP_QUERY" postgres`
do
- if [ $ENABLE_PLAIN_BACKUPS = "yes" ]
+ if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
then
echo "Plain backup of $DATABASE"
@@ -123,7 +123,7 @@ do
fi
fi
- if [ $ENABLE_CUSTOM_BACKUPS = "yes" ]
+ if [[ $ENABLE_CUSTOM_BACKUPS == "yes" ]]
then
echo "Custom backup of $DATABASE"
diff --git a/pg_backup_rotated.sh b/pg_backup_rotated.sh
index 10e36d6..7380a5e 100644
--- a/pg_backup_rotated.sh
+++ b/pg_backup_rotated.sh
@@ -4,7 +4,7 @@
####### LOAD CONFIG #######
###########################
-while [ $# -gt 0 ]; do
+while [[ "$#" -gt "0" ]]; do
case $1 in
-c)
CONFIG_FILE_PATH="$2"
@@ -17,12 +17,12 @@ while [ $# -gt 0 ]; do
esac
done
-if [ -z $CONFIG_FILE_PATH ] ; then
+if [[ -z "$CONFIG_FILE_PATH" ]]; then
SCRIPTPATH=$(cd ${0%/*} && pwd -P)
CONFIG_FILE_PATH="${SCRIPTPATH}/pg_backup.config"
fi
-if [ ! -r ${CONFIG_FILE_PATH} ] ; then
+if [[ ! -r "${CONFIG_FILE_PATH}" ]]; then
echo "Could not load config file from ${CONFIG_FILE_PATH}" 1>&2
exit 1
fi
@@ -34,7 +34,7 @@ source "${CONFIG_FILE_PATH}"
###########################
# Make sure we're running as the required backup user
-if [ "$BACKUP_USER" != "" -a "$(id -un)" != "$BACKUP_USER" ] ; then
+if [[ "$BACKUP_USER" != "" && "$(id -un)" != "$BACKUP_USER" ]]; then
echo "This script must be run as $BACKUP_USER. Exiting." 1>&2
exit 1
fi
@@ -44,11 +44,11 @@ fi
### INITIALISE DEFAULTS ###
###########################
-if [ ! $HOSTNAME ]; then
+if [[ -z "$HOSTNAME" ]]; then
HOSTNAME="localhost"
fi;
-if [ ! $USERNAME ]; then
+if [[ -z "$USERNAME" ]]; then
USERNAME="postgres"
fi;
@@ -116,7 +116,7 @@ function perform_backups()
for DATABASE in `psql -h "$HOSTNAME" -U "$USERNAME" -At -c "$FULL_BACKUP_QUERY" postgres`
do
- if [ $ENABLE_PLAIN_BACKUPS = "yes" ]
+ if [[ "$ENABLE_PLAIN_BACKUPS" == "yes" ]]
then
echo "Plain backup of $DATABASE"
@@ -127,7 +127,7 @@ function perform_backups()
fi
fi
- if [ $ENABLE_CUSTOM_BACKUPS = "yes" ]
+ if [[ "$ENABLE_CUSTOM_BACKUPS" == "yes" ]]
then
echo "Custom backup of $DATABASE"
if ! pg_dump -Fc -h "$HOSTNAME" -U "$USERNAME" "$DATABASE" | gzip --stdout | s3cmd --reduced-redundancy put - $FINAL_BACKUP_DIR"$DATABASE".custom.in_progress --no-encrypt; then
@@ -146,7 +146,7 @@ function perform_backups()
DAY_OF_MONTH=`date +%d`
-if [ $DAY_OF_MONTH -eq 1 ];
+if [[ "$DAY_OF_MONTH" -eq "1" ]];
then
# Delete all expired monthly directories
find $BACKUP_DIR -maxdepth 1 -name "*-monthly" -exec rm -rf '{}' ';'
@@ -161,7 +161,7 @@ fi
DAY_OF_WEEK=`date +%u` #1-7 (Monday-Sunday)
EXPIRED_DAYS=`expr $((($WEEKS_TO_KEEP * 7) + 1))`
-if [ $DAY_OF_WEEK = $DAY_OF_WEEK_TO_KEEP ];
+if [[ "$DAY_OF_WEEK" -eq "$DAY_OF_WEEK_TO_KEEP" ]];
then
# Delete all expired weekly directories
find $BACKUP_DIR -maxdepth 1 -mtime +$EXPIRED_DAYS -name "*-weekly" -exec rm -rf '{}' ';'
-----------------------------------------------------------------------
hooks/post-receive
--
postgresql_auto_backup_s3
More information about the Bps-public-commit
mailing list