[Bps-public-commit] brackup branch, master, updated. 7da6e5c5a44a6dcbc108e24664eab5e7826ff654
Alex Vandiver
alexmv at bestpractical.com
Thu Jan 15 16:39:55 EST 2015
The branch, master has been updated
via 7da6e5c5a44a6dcbc108e24664eab5e7826ff654 (commit)
via 7e3686e129be3a4c2e7b7a1c54a62ad9e41e6a5c (commit)
via 48a67fa7de150114b11e3369527e0b64617fced4 (commit)
via a594c69273c21801edd23597fe6ee4e0bd610805 (commit)
via 4b5ef651e996d72f8854cddacab1a73b8e1db1b9 (commit)
via fe748d5c52276fd7f27d23638c8a189b1363aa14 (commit)
via aa14efd1806800b2b4e22713d0625f04936718fa (commit)
via 4a22795936676eb3a20ba4b209be74b196d77d50 (commit)
via ba52080b7ab1cfba3b21baca35b495bc805c03e8 (commit)
via 239dd952dc4cf32d1671bc5c620ba20dde1b3837 (commit)
via f97291d40106e2de8c7bc0e6680a4b2a604524c7 (commit)
via 71a4c948e05e386e4c78133f7d88cd0dc6e2b156 (commit)
via 1e39d8489458c616c04c42fff13177213289cfa1 (commit)
via 28c7eec3b350f7e1ff11055983a5f06ca525c7a3 (commit)
via c937eca3fe075b7ebbe8ffb10829b4fffedf0988 (commit)
via df28bf22f99ead5a85fba0d8dbf5c5ee99834550 (commit)
via f87dfd7d9e01a9854156b6c832a6682cc09271e9 (commit)
via b44c36f1d9a7a3818c03dbd55651edf3f9263d88 (commit)
via 0dcd034596869207a9bb8433b9ec4ed0077cae2b (commit)
via 1960b3a6e7a26599bea5165c2848b7d682a88cb1 (commit)
via ec2f54e68e017f2ce1a9ecab577ccb1519b33c0c (commit)
via bbc2855d26974885145c578b2e7f970a17e6a5a4 (commit)
via 48f055551f9b5385b68547d94655e35cfa74fb60 (commit)
via f84fd4d3c3218d2a09699c90c7b3a1364a9ea706 (commit)
via 1c282dfc46fcb764d0562d9bf597fd0807a95cc7 (commit)
via dd7e56726a3f72daaff949c85b7562511d6ea0bd (commit)
via 4d63d7198ae216d1563832cc9448bdc52fa6aef8 (commit)
via 905778205b5a0ee1ad212e495272f6af4a44197d (commit)
via 6ee0e22be4b1a650ca67b70593c69c43aef07fe1 (commit)
via a031cd39b04742e95321d4d52119ccfcf6e8576e (commit)
via f111561cfd97a553420ff16c15860271cc4e1e6e (commit)
via 73a006d2151958ecf778c88a9d1731e10c85f2af (commit)
via 6c244f4dfb50110a6b2dd0a89e58c169de324385 (commit)
via f86b07d4172205d37ed7f018524c609475f133d9 (commit)
from 98e86321eef84f38420ccb7ab56760a5d4fcc657 (commit)
Summary of changes:
Changes | 9 +-
MANIFEST | 3 +
Makefile.PL | 18 ++--
brackup | 9 +-
brackup-restore | 74 +++++++++-----
brackup-target | 18 ++--
brackup-verify-chunks | 156 +++++++++++++++++++++++++++++
brackup-verify-inventory | 27 ++++-
lib/Brackup/Backup.pm | 31 ++++--
lib/Brackup/BackupStats.pm | 149 ++++++++++++++++++++++-----
lib/Brackup/Config.pm | 9 +-
lib/Brackup/Decrypt.pm | 2 +-
lib/Brackup/File.pm | 5 +-
lib/Brackup/Manual/Overview.pod | 7 +-
lib/Brackup/Restore.pm | 47 +++++----
lib/Brackup/Root.pm | 15 ++-
lib/Brackup/Target.pm | 26 +++--
lib/Brackup/Test.pm | 11 +-
lib/Brackup/Util.pm | 9 +-
lib/Brackup/Webhook.pm | 61 +++++++++++
t/07-restore-conflict.t | 1 +
t/08-stats.t | 46 +++++++++
t/data-2/readonly.txt | 1 +
t/data-weird-filenames/ugly_symlink_target | 1 +
t/data/readonly.txt | 1 +
25 files changed, 615 insertions(+), 121 deletions(-)
create mode 100755 brackup-verify-chunks
create mode 100644 lib/Brackup/Webhook.pm
create mode 100644 t/08-stats.t
create mode 100644 t/data-2/readonly.txt
create mode 120000 t/data-weird-filenames/ugly_symlink_target
create mode 100644 t/data/readonly.txt
- Log -----------------------------------------------------------------
commit f86b07d4172205d37ed7f018524c609475f133d9
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Mon Nov 22 16:56:05 2010 +0000
Add load_chunk stub to Brackup::Target; minor pod tweaks.
git-svn-id: http://brackup.googlecode.com/svn/trunk@333 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/Target.pm b/lib/Brackup/Target.pm
index 0fa8880..7ee138d 100644
--- a/lib/Brackup/Target.pm
+++ b/lib/Brackup/Target.pm
@@ -45,6 +45,12 @@ sub has_chunk {
die "ERROR: has_chunk not implemented in sub-class $self";
}
+# returns a chunk reference on success, or returns false or dies otherwise
+sub load_chunk {
+ my ($self, $dig) = @_;
+ die "ERROR: load_chunk not implemented in sub-class $self";
+}
+
# returns true on success, or returns false or dies otherwise.
sub store_chunk {
my ($self, $chunk) = @_;
@@ -233,7 +239,9 @@ B<Sftp> -- see L<Brackup::Target::Sftp> for configuration details
B<Amazon> -- see L<Brackup::Target::Amazon> for configuration details
-B<Amazon> -- see L<Brackup::Target::CloudFiles> for configuration details
+B<CloudFiles> -- see L<Brackup::Target::CloudFiles> for configuration details
+
+B<Riak> -- see L<Brackup::Target::Riak> for configuration details
=item B<keep_backups>
commit 6c244f4dfb50110a6b2dd0a89e58c169de324385
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Fri Jan 28 20:15:31 2011 +0000
Add extra timestamp and debug before final flush_files in Brackup::backup.
git-svn-id: http://brackup.googlecode.com/svn/trunk@334 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/Backup.pm b/lib/Brackup/Backup.pm
index 3f8fe22..4e16cd0 100644
--- a/lib/Brackup/Backup.pm
+++ b/lib/Brackup/Backup.pm
@@ -278,8 +278,10 @@ sub backup {
}
$end_file->();
$comp_chunk->finalize if $comp_chunk;
- $self->flush_files($metafh);
$stats->timestamp('Chunk Storage');
+ $self->debug('Flushing files to metafile');
+ $self->flush_files($metafh);
+ $stats->timestamp('Metafile Final Flush');
$stats->set('Number of Files Uploaded:', $n_files_up);
$stats->set('Total File Size Uploaded:', sprintf('%0.01f MB', $n_kb_up / 1024));
commit 73a006d2151958ecf778c88a9d1731e10c85f2af
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Tue Feb 1 15:41:32 2011 +0000
Make Brackup::Test::dir_structure and t/07-restore-conflict.t ignore .svn directories.
git-svn-id: http://brackup.googlecode.com/svn/trunk@335 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/Test.pm b/lib/Brackup/Test.pm
index 947caa9..3ef684c 100644
--- a/lib/Brackup/Test.pm
+++ b/lib/Brackup/Test.pm
@@ -244,7 +244,7 @@ sub dir_structure {
find({
no_chdir => 1,
- preprocess => sub { return sort @_ },
+ preprocess => sub { return sort grep ! /^\.svn$/, @_ },
wanted => sub {
my $path = $_;
$files{$path} = file_meta($path);
diff --git a/t/07-restore-conflict.t b/t/07-restore-conflict.t
index 45c38f6..79df78a 100644
--- a/t/07-restore-conflict.t
+++ b/t/07-restore-conflict.t
@@ -19,6 +19,7 @@ my $backup_file = do_backup(
$csec->add("path", $root_dir);
$csec->add("chunk_size", "2k");
$csec->add("digestdb_file", $digdb_fn);
+ $csec->add("ignore", "(^|/)\.svn/");
},
);
commit f111561cfd97a553420ff16c15860271cc4e1e6e
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Tue Feb 1 15:55:41 2011 +0000
Add readonly test data files.
git-svn-id: http://brackup.googlecode.com/svn/trunk@336 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/t/data-2/readonly.txt b/t/data-2/readonly.txt
new file mode 100644
index 0000000..37e0398
--- /dev/null
+++ b/t/data-2/readonly.txt
@@ -0,0 +1 @@
+This file should be readonly (0444).
diff --git a/t/data/readonly.txt b/t/data/readonly.txt
new file mode 100644
index 0000000..37e0398
--- /dev/null
+++ b/t/data/readonly.txt
@@ -0,0 +1 @@
+This file should be readonly (0444).
commit a031cd39b04742e95321d4d52119ccfcf6e8576e
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Tue Feb 1 15:57:19 2011 +0000
Add unlink in Brackup::Restore::_restore_file to fix readonly overwrite failure.
git-svn-id: http://brackup.googlecode.com/svn/trunk@337 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/Restore.pm b/lib/Brackup/Restore.pm
index 1181a1e..c04ca97 100644
--- a/lib/Brackup/Restore.pm
+++ b/lib/Brackup/Restore.pm
@@ -324,6 +324,8 @@ sub _restore_file {
unless $self->{conflict};
return if $self->_can_skip($full, $it);
}
+ # If $full exists, unlink (in case readonly when overwriting would fail)
+ unlink $full if -e $full;
sysopen(my $fh, $full, O_CREAT|O_WRONLY|O_TRUNC) or die "Failed to open '$full' for writing: $!";
binmode($fh);
commit 6ee0e22be4b1a650ca67b70593c69c43aef07fe1
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Tue Jul 12 07:48:19 2011 +0000
Escape and unescape link targets in the same way as paths.
git-svn-id: http://brackup.googlecode.com/svn/trunk@338 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/File.pm b/lib/Brackup/File.pm
index b428ae5..59c06f9 100644
--- a/lib/Brackup/File.pm
+++ b/lib/Brackup/File.pm
@@ -217,7 +217,7 @@ sub as_rfc822 {
} else {
$set->("Type", $type);
if ($self->is_link) {
- $set->("Link", $self->link_target);
+ $set->("Link", printable($self->link_target));
}
}
$set->("Chunks", join("\n ", map { $_->to_meta } @$schunk_list));
diff --git a/lib/Brackup/Restore.pm b/lib/Brackup/Restore.pm
index c04ca97..8151e41 100644
--- a/lib/Brackup/Restore.pm
+++ b/lib/Brackup/Restore.pm
@@ -294,7 +294,8 @@ sub _restore_link {
or die "Failed to unlink link $full: $!";
}
- symlink $it->{Link}, $full or
+ my $link = unprintable($it->{Link});
+ symlink $link, $full or
die "Failed to link $full: $!";
}
diff --git a/t/data-weird-filenames/ugly_symlink_target b/t/data-weird-filenames/ugly_symlink_target
new file mode 120000
index 0000000..fa5d1db
--- /dev/null
+++ b/t/data-weird-filenames/ugly_symlink_target
@@ -0,0 +1 @@
+uglysymlinktarget
\ No newline at end of file
commit 905778205b5a0ee1ad212e495272f6af4a44197d
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Tue Oct 11 01:21:06 2011 +0000
Allow Target prune keep_backups == 0 if --source is set.
git-svn-id: http://brackup.googlecode.com/svn/trunk@339 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/Target.pm b/lib/Brackup/Target.pm
index 7ee138d..4089b9d 100644
--- a/lib/Brackup/Target.pm
+++ b/lib/Brackup/Target.pm
@@ -116,10 +116,10 @@ sub delete_backup {
sub prune {
my ($self, %opt) = @_;
- my $keep_backups = $opt{keep_backups} || $self->{keep_backups}
- or die "ERROR: keep_backups option not set\n";
+ my $keep_backups = $opt{keep_backups} || $self->{keep_backups};
+ die "ERROR: keep_backups option not set\n" if ! defined $keep_backups;
die "ERROR: keep_backups option must be at least 1\n"
- unless $keep_backups > 0;
+ unless $keep_backups > 0 || $opt{source};
# select backups to delete
my (%backups, @backups_to_delete) = ();
commit 4d63d7198ae216d1563832cc9448bdc52fa6aef8
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Tue Oct 11 01:32:40 2011 +0000
Make brackup-target set --verbose if --dry-run set.
git-svn-id: http://brackup.googlecode.com/svn/trunk@340 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/brackup-target b/brackup-target
index ad48551..91f2817 100755
--- a/brackup-target
+++ b/brackup-target
@@ -106,7 +106,7 @@ if ($opt_help) {
Pod::Usage::pod2usage( -verbose => 1, -exitval => 0 );
exit 0;
}
-$opt_verbose = 1 if $opt_interactive && ! $opt_verbose;
+$opt_verbose ||= 1 if $opt_interactive || $opt_dryrun;
my $config = eval { Brackup::Config->load($config_file) } or
usage($@);
commit dd7e56726a3f72daaff949c85b7562511d6ea0bd
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Thu Nov 24 05:12:36 2011 +0000
Sanity check inv chunk count in brackup-verify-inventory --delete mode.
git-svn-id: http://brackup.googlecode.com/svn/trunk@341 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/brackup-verify-inventory b/brackup-verify-inventory
index f7423f2..98b93ac 100755
--- a/brackup-verify-inventory
+++ b/brackup-verify-inventory
@@ -113,6 +113,12 @@ my $inv_db = $target->inventory_db
print "Fetching list of chunks from target\n" if $opt_verbose;
my %chunks = map { $_ => 1 } $target->chunks;
+my $chunk_count = scalar keys %chunks;
+
+# Sanity check if in delete mode - abort if chunk mismatch > 10%
+die sprintf("Error: low target chunk count (%d chunks < 90% of %d inventory entries) - aborting\n",
+ $chunk_count, $inv_db->count)
+ if $opt_delete && $chunk_count < 0.9 * $inv_db->count;
print "Checking inventory entries\n" if $opt_verbose;
my ($count, $ok, $bad, $skip) = (0, 0, 0, 0);
commit 1c282dfc46fcb764d0562d9bf597fd0807a95cc7
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Thu Nov 24 05:54:02 2011 +0000
Add EXLOCK => 0 flag to File::Temp::tempfile uses, fixing cpan bug #28373.
git-svn-id: http://brackup.googlecode.com/svn/trunk@342 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/Makefile.PL b/Makefile.PL
index 1dad958..90417df 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -12,7 +12,7 @@ WriteMakefile( NAME => 'Brackup',
'DBI' => 0,
'String::Escape' => 0,
'IO::InnerFile' => 0,
- 'File::Temp' => 0.17, # require a seekable File::Temp
+ 'File::Temp' => 0.19, # require a seekable File::Temp + EXLOCK support
'ExtUtils::Manifest' => 1.52, # For spaces in files in MANIFEST
'Test::More' => 0.88, # For done_testing
},
diff --git a/lib/Brackup/Util.pm b/lib/Brackup/Util.pm
index 9e864dd..b84855d 100644
--- a/lib/Brackup/Util.pm
+++ b/lib/Brackup/Util.pm
@@ -34,12 +34,17 @@ sub _get_temp_directory {
}
sub tempfile {
- my (@ret) = File::Temp::tempfile(DIR => _get_temp_directory());
+ my (@ret) = File::Temp::tempfile(DIR => _get_temp_directory(),
+ EXLOCK => 0,
+ );
return wantarray ? @ret : $ret[0];
}
sub tempfile_obj {
- return File::Temp->new(DIR => _get_temp_directory(), CLEANUP => $ENV{BRACKUP_TEST_NOCLEANUP} ? 0 : 1);
+ return File::Temp->new(DIR => _get_temp_directory(),
+ EXLOCK => 0,
+ UNLINK => $ENV{BRACKUP_TEST_NOCLEANUP} ? 0 : 1,
+ );
}
# Utils::tempdir() accepts the same options as File::Temp::tempdir.
commit f84fd4d3c3218d2a09699c90c7b3a1364a9ea706
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Thu Nov 24 22:10:13 2011 +0000
Add explicit 'use 5.006' to Makefile.PL.
git-svn-id: http://brackup.googlecode.com/svn/trunk@343 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/Makefile.PL b/Makefile.PL
index 90417df..e8ad1cb 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -1,4 +1,5 @@
#!/usr/bin/perl
+use 5.006;
use strict;
use ExtUtils::MakeMaker;
commit 48f055551f9b5385b68547d94655e35cfa74fb60
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Wed Nov 30 03:12:48 2011 +0000
Tweak BackupStats line format.
git-svn-id: http://brackup.googlecode.com/svn/trunk@344 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/BackupStats.pm b/lib/Brackup/BackupStats.pm
index d9d0371..8e807af 100644
--- a/lib/Brackup/BackupStats.pm
+++ b/lib/Brackup/BackupStats.pm
@@ -43,7 +43,7 @@ sub print {
my $start_time = $self->{start_time};
my $end_time = time;
- my $fmt = "${hash}%-37s %s\n";
+ my $fmt = "${hash}%-39s %s\n";
printf $fh $fmt, 'Start Time:', scalar localtime $start_time;
printf $fh $fmt, 'End Time:', scalar localtime $end_time;
commit bbc2855d26974885145c578b2e7f970a17e6a5a4
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Wed Feb 29 03:42:56 2012 +0000
Croak on failed lstat in Brackup::File.
git-svn-id: http://brackup.googlecode.com/svn/trunk@345 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/File.pm b/lib/Brackup/File.pm
index 59c06f9..4b3e0e3 100644
--- a/lib/Brackup/File.pm
+++ b/lib/Brackup/File.pm
@@ -39,7 +39,8 @@ sub stat {
my $self = shift;
return $self->{stat} if $self->{stat};
my $path = $self->fullpath;
- my $stat = File::stat::lstat($path);
+ my $stat = File::stat::lstat($path)
+ or croak "Failed to lstat '$path': $!";
return $self->{stat} = $stat;
}
commit ec2f54e68e017f2ce1a9ecab577ccb1519b33c0c
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Wed Feb 29 03:45:27 2012 +0000
BackupStats cleanups, add as_hash method and unit test.
git-svn-id: http://brackup.googlecode.com/svn/trunk@346 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/MANIFEST b/MANIFEST
index d511e0d..e090098 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -70,6 +70,7 @@ t/04-gc-sftp.t
t/05-filename-escaping.t
t/06-config-inheritance.t
t/07-restore-conflict.t
+t/08-stats.t
t/data-2/000-dup1.txt
t/data-2/000-dup2.txt
t/data-2/README
diff --git a/brackup b/brackup
index 45e0274..ae8fe2d 100755
--- a/brackup
+++ b/brackup
@@ -206,12 +206,12 @@ my $backup = Brackup::Backup->new(
dryrun => $opt_dryrun,
verbose => $opt_verbose,
zenityprogress => $opt_zenityprogress,
+ arguments => $arguments,
);
if (my $stats = eval { $backup->backup($backup_file) }) {
warn "Backup complete.\n" if $opt_verbose;
- $stats->set('Run Arguments:' => $arguments);
if ($opt_dryrun || $opt_verbose) {
$stats->print;
}
diff --git a/lib/Brackup/Backup.pm b/lib/Brackup/Backup.pm
index 4e16cd0..14164c9 100644
--- a/lib/Brackup/Backup.pm
+++ b/lib/Brackup/Backup.pm
@@ -20,6 +20,7 @@ sub new {
$self->{inventory} = delete $opts{inventory}; # bool
$self->{savefiles} = delete $opts{savefiles}; # bool
$self->{zenityprogress} = delete $opts{zenityprogress}; # bool
+ $self->{arguments} = delete $opts{arguments};
$self->{modecounts} = {}; # type -> mode(octal) -> count
$self->{idcounts} = {}; # type -> uid/gid -> count
@@ -42,7 +43,7 @@ sub backup {
my $root = $self->{root};
my $target = $self->{target};
- my $stats = Brackup::BackupStats->new;
+ my $stats = Brackup::BackupStats->new(arguments => $self->{arguments});
my @gpg_rcpts = $self->{root}->gpg_rcpts;
@@ -71,8 +72,8 @@ sub backup {
$self->debug("Number of files: $n_files\n");
$stats->timestamp('File Discovery');
- $stats->set('Number of Files' => $n_files);
- $stats->set('Total File Size' => sprintf('%0.01f MB', $n_kb / 1024));
+ $stats->set(files_checked_count => $n_files, label => 'Number of Files');
+ $stats->set(files_checked_size => sprintf('%0.01f', $n_kb / 1024), label => 'Total File Size', units => 'MB');
# calc needed chunks
if ($ENV{CALC_NEEDED}) {
@@ -282,8 +283,8 @@ sub backup {
$self->debug('Flushing files to metafile');
$self->flush_files($metafh);
$stats->timestamp('Metafile Final Flush');
- $stats->set('Number of Files Uploaded:', $n_files_up);
- $stats->set('Total File Size Uploaded:', sprintf('%0.01f MB', $n_kb_up / 1024));
+ $stats->set(files_uploaded_count => $n_files_up, label => 'Number of Files Uploaded');
+ $stats->set(files_uploaded_size => sprintf('%0.01f', $n_kb_up / 1024), label => 'Total File Size Uploaded', units => 'MB');
unless ($self->{dryrun}) {
close $metafh or die "Close on metafile '$backup_file' failed: $!";
diff --git a/lib/Brackup/BackupStats.pm b/lib/Brackup/BackupStats.pm
index 8e807af..f92cb23 100644
--- a/lib/Brackup/BackupStats.pm
+++ b/lib/Brackup/BackupStats.pm
@@ -1,21 +1,24 @@
package Brackup::BackupStats;
use strict;
+use Carp;
sub new {
my $class = shift;
my %opts = @_;
+ my $arguments = delete $opts{arguments};
croak("Unknown options: " . join(', ', keys %opts)) if %opts;
my $self = {
- start_time => time,
- ts => Brackup::BackupStats::Data->new,
- data => Brackup::BackupStats::Data->new,
+ start_time => time,
+ ts => Brackup::BackupStats::OrderedData->new,
+ data => Brackup::BackupStats::LabelledData->new,
};
+ $self->{arguments} = $arguments if $arguments;
if (eval { require GTop }) {
$self->{gtop} = GTop->new;
$self->{gtop_max} = 0;
- $self->{gtop_data} = Brackup::BackupStats::Data->new;
+ $self->{gtop_data} = Brackup::BackupStats::OrderedData->new;
}
return bless $self, $class;
@@ -24,6 +27,8 @@ sub new {
sub print {
my $self = shift;
my $stats_file = shift;
+
+ $self->end;
# Reset iterators
$self->reset;
@@ -42,13 +47,16 @@ sub print {
print $fh "${hash}\n";
my $start_time = $self->{start_time};
- my $end_time = time;
+ my $end_time = $self->{end_time};
my $fmt = "${hash}%-39s %s\n";
printf $fh $fmt, 'Start Time:', scalar localtime $start_time;
printf $fh $fmt, 'End Time:', scalar localtime $end_time;
+ printf $fh $fmt, 'Run Arguments:', $self->{arguments} if $self->{arguments};
+ printf $fh "${hash}\n";
my $ts = $start_time;
- while (my ($label, $next_ts) = $self->{ts}->next) {
+ while (my ($key, $next_ts, $label) = $self->{ts}->next) {
+ $label ||= $key;
printf $fh $fmt, "$label Time:", ($next_ts - $ts) . 's';
$ts = $next_ts;
}
@@ -56,7 +64,8 @@ sub print {
print $fh "${hash}\n";
if (my $gtop_data = $self->{gtop_data}) {
- while (my ($label, $size) = $gtop_data->next) {
+ while (my ($key, $size, $label) = $gtop_data->next) {
+ $label ||= $key;
printf $fh $fmt,
"Post $label Memory Usage:", sprintf('%0.1f MB', $size / (1024 * 1024));
}
@@ -69,10 +78,64 @@ sub print {
}
my $data = $self->{data};
- while (my ($key, $value) = $data->next) {
- printf $fh $fmt, $key, $value;
+ for my $key (sort keys %$data) {
+ my $value = $data->{$key}->{value};
+ $value .= " $data->{$key}->{units}" if $data->{$key}->{units};
+ my $label = $data->{$key}->{label} || $key;
+ $label .= ':' if substr($label,-1) ne ':';
+ printf $fh $fmt, $label, $value;
+ }
+ printf $fh "${hash}\n";
+}
+
+# Return a hashref of the stats data
+sub as_hash {
+ my $self = shift;
+
+ $self->end;
+
+ # Reset iterators
+ $self->reset;
+
+ my $hash = {
+ start_time => $self->{start_time},
+ end_time => $self->{end_time},
+ run_times => [],
+ memory_usage => [],
+ files => {},
+ };
+ $hash->{arguments} = $self->{arguments} if $self->{arguments};
+
+ # Run time stats (seconds)
+ my $ts = $self->{start_time};
+ while (my ($key, $next_ts, $label) = $self->{ts}->next) {
+ push @{$hash->{run_times}}, $key => ($next_ts - $ts) . ' s';
+ $ts = $next_ts;
+ }
+ push @{$hash->{run_times}}, "total" => ($self->{end_time} - $self->{start_time}) . ' s';
+
+ # Memory stats(MB)
+ if (my $gtop_data = $self->{gtop_data}) {
+ while (my ($key, $size, $label) = $gtop_data->next) {
+ $size /= (1024 * 1024);
+ push @{$hash->{memory_usage}}, "post_$key" => sprintf('%.01f MB', $size);
+ }
}
- print $fh "\n" if $stats_file;
+ push @{$hash->{memory_usage}}, max => sprintf('%.01f MB', $self->{gtop_max} / (1024 * 1024));
+
+ # File stats (hashref, key => value)
+ for (keys %{$self->{data}}) {
+ $hash->{files}->{$_} = $self->{data}->{$_}->{value};
+ $hash->{files}->{$_} .= " $self->{data}->{$_}->{units}" if $self->{data}->{$_}->{units};
+ }
+
+ return $hash;
+}
+
+# Record end time
+sub end {
+ my $self = shift;
+ $self->{end_time} ||= time;
}
# Check/record max memory usage
@@ -83,24 +146,31 @@ sub check_maxmem {
$self->{gtop_max} = $mem if $mem > $self->{gtop_max};
}
-# Record current time (and memory, if applicable) against $label
+# Record current time (and memory, if applicable) against $key in ts dataset
sub timestamp {
- my ($self, $label) = @_;
- $self->{ts}->set($label => time);
+ my ($self, $key, $label) = @_;
+ # If no label given, assume key is actually label, and derive key
+ if (! $label) {
+ $label = $key;
+ $key = lc $label;
+ $key =~ s/\s+/_/g
+ }
+ $self->{ts}->set($key => time, label => $label);
return unless $self->{gtop};
- $self->{gtop_data}->set($label => $self->{gtop}->proc_mem($$)->size);
+ $self->{gtop_data}->set($key => $self->{gtop}->proc_mem($$)->size, label => $label);
$self->check_maxmem;
}
+# Record a datum
sub set {
- my $self = shift;
- $self->{data}->set(shift, shift) while @_ >= 2;
+ my ($self, $key, $value, %arg) = @_;
+ $self->{data}->set($key, $value, %arg);
}
+# Reset dataset iterators
sub reset {
my $self = shift;
$self->{ts}->reset;
- $self->{data}->reset;
$self->{gtop_data}->reset if $self->{gtop_data};
}
@@ -108,30 +178,34 @@ sub note_stored_chunk {
my ($self, $chunk) = @_;
}
-package Brackup::BackupStats::Data;
+package Brackup::BackupStats::OrderedData;
+
+use Carp;
sub new {
my $class = shift;
return bless {
- index => 0,
- list => [], # ordered list of data keys
- data => {},
+ index => 0,
+ list => [], # ordered list of data keys
+ data => {},
+ label => {},
}, $class;
}
sub set {
- my ($self, $key, $value) = @_;
+ my ($self, $key, $value, %arg) = @_;
die "data key '$key' exists" if exists $self->{data}->{$key};
push @{$self->{list}}, $key;
- $self->{data}->{$key} = $value;
+ $self->{data}->{$key} = $value;
+ $self->{label}->{$key} = $arg{label} if $arg{label};
}
-# Iterator interface, returning ($key, $value)
+# Iterator interface, returning ($key, $value, $label)
sub next {
my $self = shift;
return () unless $self->{index} <= $#{$self->{list}};
my $key = $self->{list}->[$self->{index}++];
- return ($key, $self->{data}->{$key});
+ return ($key, $self->{data}->{$key}, $self->{label}->{$key});
}
# Reset/rewind iterator
@@ -140,6 +214,21 @@ sub reset {
$self->{index} = 0;
}
+package Brackup::BackupStats::LabelledData;
+
+sub new {
+ my $class = shift;
+ return bless {}, $class;
+}
+
+sub set {
+ my ($self, $key, $value, %arg) = @_;
+ die "data key '$key' exists" if exists $self->{$key};
+ $self->{$key} = { value => $value };
+ $self->{$key}->{label} = $arg{label} if $arg{label};
+ $self->{$key}->{units} = $arg{units} if $arg{units};
+}
+
1;
# vim:sw=4
diff --git a/lib/Brackup/Test.pm b/lib/Brackup/Test.pm
index 3ef684c..8329ab0 100644
--- a/lib/Brackup/Test.pm
+++ b/lib/Brackup/Test.pm
@@ -36,6 +36,7 @@ sub do_backup {
my $with_targetsec = delete $opts{'with_targetsec'} || sub {};
my $with_root = delete $opts{'with_root'} || sub {};
my $target = delete $opts{'with_target'};
+ my $arguments = delete $opts{'with_arguments'};
die if %opts;
my $initer = shift;
@@ -75,6 +76,7 @@ sub do_backup {
root => $root,
target => $target,
savefiles => 1,
+ arguments => $arguments,
);
ok($backup, "have a backup object");
@@ -82,7 +84,8 @@ sub do_backup {
ok(-e $meta_filename, "metafile exists");
push @to_unlink, $meta_filename;
- ok(eval { $backup->backup($meta_filename) }, "backup succeeded");
+ my $stats;
+ ok($stats = eval { $backup->backup($meta_filename) }, "backup succeeded");
if ($@) {
warn "Died running backup: $@\n";
}
@@ -90,7 +93,7 @@ sub do_backup {
check_inventory_db($target, [$root->gpg_args]);
- return wantarray ? ($meta_filename, $backup, $target) : $meta_filename;
+ return wantarray ? ($meta_filename, $backup, $target, $stats) : $meta_filename;
}
sub check_inventory_db {
diff --git a/t/08-stats.t b/t/08-stats.t
new file mode 100644
index 0000000..a0b2451
--- /dev/null
+++ b/t/08-stats.t
@@ -0,0 +1,45 @@
+# -*-perl-*-
+
+use strict;
+use Test::More;
+
+use Brackup::Test;
+use FindBin qw($Bin);
+use Brackup::Util qw(tempfile);
+
+############### Backup
+
+my $arguments = '--from test_root --to test_target -v';
+my ($digdb_fh, $digdb_fn) = tempfile();
+close($digdb_fh);
+my $root_dir = "$Bin/data";
+ok(-d $root_dir, "test data to backup exists");
+my ($backup_file, $brackup, $target, $stats) = do_backup(
+ with_confsec => sub {
+ my $csec = shift;
+ $csec->add("path", $root_dir);
+ $csec->add("chunk_size", "2k");
+ $csec->add("digestdb_file", $digdb_fn);
+ $csec->add("webhook_url", $ENV{BRACKUP_TEST_WEBHOOK_URL})
+ if $ENV{BRACKUP_TEST_WEBHOOK_URL};
+ },
+ with_arguments => $arguments,
+ );
+
+my $stats_hash = $stats->as_hash;
+is($stats_hash->{arguments}, $arguments, '$stats_hash arguments set correctly');
+for (qw(start_time end_time run_times memory_usage files)) {
+ ok($stats_hash->{$_}, "\$stats_hash $_ set");
+}
+for (qw(run_times memory_usage)) {
+ ok(@{$stats_hash->{$_}}, "\$stats_hash $_ count non-zero: " . scalar(@{$stats_hash->{$_}}));
+}
+is($stats_hash->{files}->{files_checked_count}, 17, 'stats files_checked_count ok: ' . $stats_hash->{files}->{files_checked_count});
+is($stats_hash->{files}->{files_uploaded_count}, 11, 'stats files_uploaded_count ok: ' . $stats_hash->{files}->{files_uploaded_count});
+
+#use Data::Dump qw(pp);
+#pp $stats_hash;
+#$stats->print;
+
+done_testing;
+
commit 1960b3a6e7a26599bea5165c2848b7d682a88cb1
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Wed Feb 29 03:48:04 2012 +0000
Add simple webhook support.
git-svn-id: http://brackup.googlecode.com/svn/trunk@347 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/MANIFEST b/MANIFEST
index e090098..3ccd30d 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -55,6 +55,7 @@ lib/Brackup/Target/Sftp.pm
lib/Brackup/TargetBackupStatInfo.pm
lib/Brackup/Test.pm
lib/Brackup/Util.pm
+lib/Brackup/Webhook.pm
t/00-use.t
t/01-backup-filesystem.t
t/01-backup-ftp.t
diff --git a/brackup b/brackup
index ae8fe2d..45e0274 100755
--- a/brackup
+++ b/brackup
@@ -206,12 +206,12 @@ my $backup = Brackup::Backup->new(
dryrun => $opt_dryrun,
verbose => $opt_verbose,
zenityprogress => $opt_zenityprogress,
- arguments => $arguments,
);
if (my $stats = eval { $backup->backup($backup_file) }) {
warn "Backup complete.\n" if $opt_verbose;
+ $stats->set('Run Arguments:' => $arguments);
if ($opt_dryrun || $opt_verbose) {
$stats->print;
}
diff --git a/lib/Brackup/Backup.pm b/lib/Brackup/Backup.pm
index 14164c9..57815ea 100644
--- a/lib/Brackup/Backup.pm
+++ b/lib/Brackup/Backup.pm
@@ -6,6 +6,7 @@ use Brackup::ChunkIterator;
use Brackup::CompositeChunk;
use Brackup::GPGProcManager;
use Brackup::GPGProcess;
+use Brackup::Webhook;
use File::Basename;
use File::Temp qw(tempfile);
@@ -20,7 +21,6 @@ sub new {
$self->{inventory} = delete $opts{inventory}; # bool
$self->{savefiles} = delete $opts{savefiles}; # bool
$self->{zenityprogress} = delete $opts{zenityprogress}; # bool
- $self->{arguments} = delete $opts{arguments};
$self->{modecounts} = {}; # type -> mode(octal) -> count
$self->{idcounts} = {}; # type -> uid/gid -> count
@@ -43,7 +43,7 @@ sub backup {
my $root = $self->{root};
my $target = $self->{target};
- my $stats = Brackup::BackupStats->new(arguments => $self->{arguments});
+ my $stats = Brackup::BackupStats->new;
my @gpg_rcpts = $self->{root}->gpg_rcpts;
@@ -330,6 +330,10 @@ sub backup {
}
$self->report_progress(100, "Backup complete.");
+ if (my $url = $root->webhook_url) {
+ Brackup::Webhook->new(url => $url, root => $root, target => $target, stats => $stats)->fire;
+ }
+
return $stats;
}
diff --git a/lib/Brackup/BackupStats.pm b/lib/Brackup/BackupStats.pm
index f92cb23..0d12711 100644
--- a/lib/Brackup/BackupStats.pm
+++ b/lib/Brackup/BackupStats.pm
@@ -1,11 +1,9 @@
package Brackup::BackupStats;
use strict;
-use Carp;
sub new {
my $class = shift;
my %opts = @_;
- my $arguments = delete $opts{arguments};
croak("Unknown options: " . join(', ', keys %opts)) if %opts;
my $self = {
@@ -13,7 +11,6 @@ sub new {
ts => Brackup::BackupStats::OrderedData->new,
data => Brackup::BackupStats::LabelledData->new,
};
- $self->{arguments} = $arguments if $arguments;
if (eval { require GTop }) {
$self->{gtop} = GTop->new;
@@ -27,8 +24,6 @@ sub new {
sub print {
my $self = shift;
my $stats_file = shift;
-
- $self->end;
# Reset iterators
$self->reset;
@@ -47,7 +42,7 @@ sub print {
print $fh "${hash}\n";
my $start_time = $self->{start_time};
- my $end_time = $self->{end_time};
+ my $end_time = time;
my $fmt = "${hash}%-39s %s\n";
printf $fh $fmt, 'Start Time:', scalar localtime $start_time;
printf $fh $fmt, 'End Time:', scalar localtime $end_time;
@@ -132,12 +127,6 @@ sub as_hash {
return $hash;
}
-# Record end time
-sub end {
- my $self = shift;
- $self->{end_time} ||= time;
-}
-
# Check/record max memory usage
sub check_maxmem {
my $self = shift;
@@ -161,7 +150,6 @@ sub timestamp {
$self->check_maxmem;
}
-# Record a datum
sub set {
my ($self, $key, $value, %arg) = @_;
$self->{data}->set($key, $value, %arg);
diff --git a/lib/Brackup/Root.pm b/lib/Brackup/Root.pm
index 642a2de..04f0751 100644
--- a/lib/Brackup/Root.pm
+++ b/lib/Brackup/Root.pm
@@ -36,12 +36,16 @@ sub new {
$self->{digcache_file} = $self->{digcache}->backing_file; # may be empty, if digest cache doesn't use a file
$self->{noatime} = $conf->value('noatime');
+
+ $self->{webhook_url} = $conf->value('webhook_url');
+
return $self;
}
sub merge_files_under { $_[0]{merge_files_under} }
sub max_composite_size { $_[0]{max_composite_size} }
sub smart_mp3_chunking { $_[0]{smart_mp3_chunking} }
+sub webhook_url { $_[0]{webhook_url} }
sub gpg_path {
my $self = shift;
@@ -254,6 +258,7 @@ In your ~/.brackup.conf file:
ignore = ^\.ee/(minis|icons|previews)/
ignore = ^build/
noatime = 1
+ webhook_url = http://example.com/hook
=head1 CONFIG OPTIONS
@@ -325,4 +330,11 @@ The example above could also be written:
ignore = ^build/
noatime = 1
+=item B<webhook_url>
+
+URL to be POSTed to upon backup completion. The post payload is a json
+object with 'root', 'target', and 'stats' members, with the first two
+being the source and target name strings, and 'stats' being a serialised
+L<Brackup::BackupStats> object.
+
=back
diff --git a/lib/Brackup/Webhook.pm b/lib/Brackup/Webhook.pm
new file mode 100644
index 0000000..3274b35
--- /dev/null
+++ b/lib/Brackup/Webhook.pm
@@ -0,0 +1,57 @@
+package Brackup::Webhook;
+use strict;
+use warnings;
+use Carp qw(croak);
+use JSON;
+use LWP::UserAgent;
+use Data::Dump;
+
+sub new {
+ my ($class, %opts) = @_;
+ my $self = bless {}, $class;
+
+ $self->{url} = delete $opts{url}
+ or croak "Missing required 'url' argument";
+ $self->{root} = delete $opts{root}
+ or croak "Missing required 'root' argument";
+ $self->{target} = delete $opts{target}
+ or croak "Missing required 'target' argument";
+ $self->{stats} = delete $opts{stats}
+ or croak "Missing required 'stats' argument";
+
+ $self->{ua} = delete $opts{user_agent} || LWP::UserAgent->new(env_proxy => 1);
+
+ croak("Unknown options: " . join(', ', keys %opts)) if %opts;
+
+ $self->{data} = {
+ root => $self->{root}->name,
+ target => $self->{target}->name,
+ stats => $self->{stats}->as_hash,
+ };
+
+ return $self;
+}
+
+sub fire {
+ my $self = shift;
+
+ my $resp = $self->{ua}->post($self->{url},
+ 'Content-Type' => 'application/json',
+ Content => encode_json($self->{data}),
+ );
+
+ # Just warn on failure?
+ $resp->is_success
+ or warn "webhook failure: " . $resp->code . ' ' . $resp->status_line;
+
+ return $resp->code >= 200 && $resp->code <= 299;
+}
+
+sub dump {
+ Data::Dump::dump $_[0]->{data};
+}
+
+1;
+
+# vim:sw=4
+
commit 0dcd034596869207a9bb8433b9ec4ed0077cae2b
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Wed Feb 29 04:19:40 2012 +0000
Fix broken patch from webhook merge.
git-svn-id: http://brackup.googlecode.com/svn/trunk@348 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/brackup b/brackup
index 45e0274..ae8fe2d 100755
--- a/brackup
+++ b/brackup
@@ -206,12 +206,12 @@ my $backup = Brackup::Backup->new(
dryrun => $opt_dryrun,
verbose => $opt_verbose,
zenityprogress => $opt_zenityprogress,
+ arguments => $arguments,
);
if (my $stats = eval { $backup->backup($backup_file) }) {
warn "Backup complete.\n" if $opt_verbose;
- $stats->set('Run Arguments:' => $arguments);
if ($opt_dryrun || $opt_verbose) {
$stats->print;
}
diff --git a/lib/Brackup/Backup.pm b/lib/Brackup/Backup.pm
index 57815ea..fd9b5c3 100644
--- a/lib/Brackup/Backup.pm
+++ b/lib/Brackup/Backup.pm
@@ -21,6 +21,7 @@ sub new {
$self->{inventory} = delete $opts{inventory}; # bool
$self->{savefiles} = delete $opts{savefiles}; # bool
$self->{zenityprogress} = delete $opts{zenityprogress}; # bool
+ $self->{arguments} = delete $opts{arguments};
$self->{modecounts} = {}; # type -> mode(octal) -> count
$self->{idcounts} = {}; # type -> uid/gid -> count
@@ -43,7 +44,7 @@ sub backup {
my $root = $self->{root};
my $target = $self->{target};
- my $stats = Brackup::BackupStats->new;
+ my $stats = Brackup::BackupStats->new(arguments => $self->{arguments});
my @gpg_rcpts = $self->{root}->gpg_rcpts;
diff --git a/lib/Brackup/BackupStats.pm b/lib/Brackup/BackupStats.pm
index 0d12711..f92cb23 100644
--- a/lib/Brackup/BackupStats.pm
+++ b/lib/Brackup/BackupStats.pm
@@ -1,9 +1,11 @@
package Brackup::BackupStats;
use strict;
+use Carp;
sub new {
my $class = shift;
my %opts = @_;
+ my $arguments = delete $opts{arguments};
croak("Unknown options: " . join(', ', keys %opts)) if %opts;
my $self = {
@@ -11,6 +13,7 @@ sub new {
ts => Brackup::BackupStats::OrderedData->new,
data => Brackup::BackupStats::LabelledData->new,
};
+ $self->{arguments} = $arguments if $arguments;
if (eval { require GTop }) {
$self->{gtop} = GTop->new;
@@ -24,6 +27,8 @@ sub new {
sub print {
my $self = shift;
my $stats_file = shift;
+
+ $self->end;
# Reset iterators
$self->reset;
@@ -42,7 +47,7 @@ sub print {
print $fh "${hash}\n";
my $start_time = $self->{start_time};
- my $end_time = time;
+ my $end_time = $self->{end_time};
my $fmt = "${hash}%-39s %s\n";
printf $fh $fmt, 'Start Time:', scalar localtime $start_time;
printf $fh $fmt, 'End Time:', scalar localtime $end_time;
@@ -127,6 +132,12 @@ sub as_hash {
return $hash;
}
+# Record end time
+sub end {
+ my $self = shift;
+ $self->{end_time} ||= time;
+}
+
# Check/record max memory usage
sub check_maxmem {
my $self = shift;
@@ -150,6 +161,7 @@ sub timestamp {
$self->check_maxmem;
}
+# Record a datum
sub set {
my ($self, $key, $value, %arg) = @_;
$self->{data}->set($key, $value, %arg);
commit b44c36f1d9a7a3818c03dbd55651edf3f9263d88
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Wed Feb 29 04:32:25 2012 +0000
Fix failing t/08-stats.t by ignoring .svn directories.
git-svn-id: http://brackup.googlecode.com/svn/trunk@349 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/t/08-stats.t b/t/08-stats.t
index a0b2451..665a330 100644
--- a/t/08-stats.t
+++ b/t/08-stats.t
@@ -20,6 +20,7 @@ my ($backup_file, $brackup, $target, $stats) = do_backup(
$csec->add("path", $root_dir);
$csec->add("chunk_size", "2k");
$csec->add("digestdb_file", $digdb_fn);
+ $csec->add("ignore", "\.svn");
$csec->add("webhook_url", $ENV{BRACKUP_TEST_WEBHOOK_URL})
if $ENV{BRACKUP_TEST_WEBHOOK_URL};
},
commit f87dfd7d9e01a9854156b6c832a6682cc09271e9
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Wed Feb 29 05:47:42 2012 +0000
Add hostname to Brackup::Webhook payload.
git-svn-id: http://brackup.googlecode.com/svn/trunk@350 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/Webhook.pm b/lib/Brackup/Webhook.pm
index 3274b35..f7fca22 100644
--- a/lib/Brackup/Webhook.pm
+++ b/lib/Brackup/Webhook.pm
@@ -4,7 +4,7 @@ use warnings;
use Carp qw(croak);
use JSON;
use LWP::UserAgent;
-use Data::Dump;
+use Sys::Hostname;
sub new {
my ($class, %opts) = @_;
@@ -24,9 +24,10 @@ sub new {
croak("Unknown options: " . join(', ', keys %opts)) if %opts;
$self->{data} = {
- root => $self->{root}->name,
- target => $self->{target}->name,
- stats => $self->{stats}->as_hash,
+ hostname => hostname,
+ root => $self->{root}->name,
+ target => $self->{target}->name,
+ stats => $self->{stats}->as_hash,
};
return $self;
@@ -48,6 +49,7 @@ sub fire {
}
sub dump {
+ require Data::Dump;
Data::Dump::dump $_[0]->{data};
}
commit df28bf22f99ead5a85fba0d8dbf5c5ee99834550
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Wed Mar 7 23:04:09 2012 +0000
Add date field to webhook data.
git-svn-id: http://brackup.googlecode.com/svn/trunk@351 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/Webhook.pm b/lib/Brackup/Webhook.pm
index f7fca22..c5d84e6 100644
--- a/lib/Brackup/Webhook.pm
+++ b/lib/Brackup/Webhook.pm
@@ -5,6 +5,7 @@ use Carp qw(croak);
use JSON;
use LWP::UserAgent;
use Sys::Hostname;
+use POSIX qw(strftime);
sub new {
my ($class, %opts) = @_;
@@ -25,6 +26,7 @@ sub new {
$self->{data} = {
hostname => hostname,
+ date => strftime('%Y%m%d', localtime),
root => $self->{root}->name,
target => $self->{target}->name,
stats => $self->{stats}->as_hash,
commit c937eca3fe075b7ebbe8ffb10829b4fffedf0988
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Wed Mar 7 23:20:07 2012 +0000
Remove units from BackupStats::as_hash numerics, add to field names.
git-svn-id: http://brackup.googlecode.com/svn/trunk@352 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/BackupStats.pm b/lib/Brackup/BackupStats.pm
index f92cb23..78c9e74 100644
--- a/lib/Brackup/BackupStats.pm
+++ b/lib/Brackup/BackupStats.pm
@@ -98,35 +98,43 @@ sub as_hash {
$self->reset;
my $hash = {
- start_time => $self->{start_time},
- end_time => $self->{end_time},
- run_times => [],
- memory_usage => [],
- files => {},
+ start_time => $self->{start_time},
+ end_time => $self->{end_time},
+ run_time_seconds => [],
+ memory_usage_bytes => [],
+ files => {},
};
$hash->{arguments} = $self->{arguments} if $self->{arguments};
# Run time stats (seconds)
my $ts = $self->{start_time};
while (my ($key, $next_ts, $label) = $self->{ts}->next) {
- push @{$hash->{run_times}}, $key => ($next_ts - $ts) . ' s';
+ push @{$hash->{run_time_seconds}}, $key => ($next_ts - $ts);
$ts = $next_ts;
}
- push @{$hash->{run_times}}, "total" => ($self->{end_time} - $self->{start_time}) . ' s';
+ push @{$hash->{run_time_seconds}}, "total" => ($self->{end_time} - $self->{start_time});
- # Memory stats(MB)
+ # Memory stats (bytes)
if (my $gtop_data = $self->{gtop_data}) {
while (my ($key, $size, $label) = $gtop_data->next) {
- $size /= (1024 * 1024);
- push @{$hash->{memory_usage}}, "post_$key" => sprintf('%.01f MB', $size);
+ push @{$hash->{memory_usage_bytes}}, "post_$key" => $size;
}
}
- push @{$hash->{memory_usage}}, max => sprintf('%.01f MB', $self->{gtop_max} / (1024 * 1024));
+ push @{$hash->{memory_usage_bytes}}, max => $self->{gtop_max};
# File stats (hashref, key => value)
for (keys %{$self->{data}}) {
- $hash->{files}->{$_} = $self->{data}->{$_}->{value};
- $hash->{files}->{$_} .= " $self->{data}->{$_}->{units}" if $self->{data}->{$_}->{units};
+ if ($self->{data}->{$_}->{units}) {
+ if ($self->{data}->{$_}->{units} eq 'MB') {
+ $hash->{files}->{$_ . '_bytes'} = $self->{data}->{$_}->{value} * 1024 * 1024;
+ }
+ else {
+ warn "Unhandled units: $self->{data}->{$_}->{units}";
+ }
+ }
+ else {
+ $hash->{files}->{$_} = $self->{data}->{$_}->{value};
+ }
}
return $hash;
diff --git a/t/08-stats.t b/t/08-stats.t
index 665a330..8962d17 100644
--- a/t/08-stats.t
+++ b/t/08-stats.t
@@ -29,10 +29,10 @@ my ($backup_file, $brackup, $target, $stats) = do_backup(
my $stats_hash = $stats->as_hash;
is($stats_hash->{arguments}, $arguments, '$stats_hash arguments set correctly');
-for (qw(start_time end_time run_times memory_usage files)) {
+for (qw(start_time end_time run_time_seconds memory_usage_bytes files)) {
ok($stats_hash->{$_}, "\$stats_hash $_ set");
}
-for (qw(run_times memory_usage)) {
+for (qw(run_time_seconds memory_usage_bytes)) {
ok(@{$stats_hash->{$_}}, "\$stats_hash $_ count non-zero: " . scalar(@{$stats_hash->{$_}}));
}
is($stats_hash->{files}->{files_checked_count}, 17, 'stats files_checked_count ok: ' . $stats_hash->{files}->{files_checked_count});
commit 28c7eec3b350f7e1ff11055983a5f06ca525c7a3
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Thu Mar 8 04:59:35 2012 +0000
Add onerror => continue support to brackup-restore.
git-svn-id: http://brackup.googlecode.com/svn/trunk@353 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/brackup-restore b/brackup-restore
index ec65886..7adb8ef 100755
--- a/brackup-restore
+++ b/brackup-restore
@@ -1,8 +1,6 @@
#!/usr/bin/perl
-# TODO: skip-if-exists (and ignore zero byte files)
-# TODO: continue-on-errors
-# Error doing restore: File restore/.brackup-digest.db (.brackup-digest.db) already exists. Aborting. at /raid/bradfitz/proj/brackup/trunk/lib/Brackup/Restore.pm line 157, <$fh> line 20.
+# TODO: onerror=prompt?
=head1 NAME
@@ -42,14 +40,22 @@ Restore just the directory named (and all its contents).
Restore just the file named.
-=item --conflict=skip|overwrite|update
+=item --onerror=abort|continue
+
+How to handle restore errors. 'abort' reports and stops as soon as an
+error is detected. 'continue' continues with the restore, collecting all
+errors and reporting them at the end of the restore.
+
+Default: abort.
+
+=item --conflict=abort|skip|overwrite|update
How to handle files that already exist (with size > zero bytes).
'skip' means don't restore, keeping the existing file. 'overwrite'
means always restore, replacing the existing file. 'update' means
overwrite iff the file we are restoring is newer than the existing one.
-Default is to abort the restore if a file already exists.
+Default: abort.
=item --config=NAME
@@ -81,6 +87,7 @@ use strict;
use warnings;
use Getopt::Long;
use File::Path;
+use Try::Tiny;
use FindBin qw($Bin);
use lib "$Bin/lib";
@@ -88,18 +95,21 @@ use lib "$Bin/lib";
use Brackup;
use Brackup::Util qw(tempfile);
-my ($opt_verbose, $meta_file, $opt_help, $restore_dir, $opt_all, $prefix, $conflict, $config_file);
+my ($opt_verbose, $meta_file, $opt_help, $restore_dir, $opt_all, $prefix, $config_file);
+my $onerror = 'abort';
+my $conflict = 'abort';
usage() unless
GetOptions(
- 'from=s' => \$meta_file,
- 'to=s' => \$restore_dir,
- 'verbose' => \$opt_verbose,
- 'help' => \$opt_help,
- 'all' => \$opt_all,
- 'just=s' => \$prefix,
- 'conflict=s'=> \$conflict,
- 'config=s' => \$config_file,
+ 'from=s' => \$meta_file,
+ 'to=s' => \$restore_dir,
+ 'verbose' => \$opt_verbose,
+ 'help' => \$opt_help,
+ 'all' => \$opt_all,
+ 'just=s' => \$prefix,
+ 'onerror|on-error=s' => \$onerror,
+ 'conflict=s' => \$conflict,
+ 'config=s' => \$config_file,
);
if ($opt_help) {
@@ -113,8 +123,10 @@ usage("Backup metafile '$meta_file' doesn't exist") unless -e $meta_file;
usage("Backup metafile '$meta_file' isn't a file") unless -f $meta_file;
usage("Restore directory '$restore_dir' isn't a directory") if -e $restore_dir && ! -d $restore_dir;
usage("Config file '$config_file' doesn't exist") if $config_file && ! -f $config_file;
-usage("Invalid --conflict option '$conflict' (not skip|overwrite|update)")
- if $conflict && $conflict !~ m/^(skip|overwrite|update)$/;
+usage("Invalid --onerror option '$onerror' (not abort|continue)")
+ if $onerror !~ m/^(abort|continue)$/;
+usage("Invalid --conflict option '$conflict' (not abort|skip|overwrite|update)")
+ if $conflict !~ m/^(abort|skip|overwrite|update)$/;
$prefix ||= ""; # with -all, "", which means everything
if (! -e $restore_dir) {
@@ -125,22 +137,30 @@ $config_file ||= Brackup::Config->default_config_file_name;
my $config = Brackup::Config->load($config_file) if -f $config_file;
my $restore = Brackup::Restore->new(
- to => $restore_dir,
- prefix => $prefix,
- file => $meta_file,
- config => $config,
- conflict => $conflict,
- verbose => $opt_verbose,
+ to => $restore_dir,
+ prefix => $prefix,
+ file => $meta_file,
+ config => $config,
+ onerror => $onerror,
+ conflict => $conflict,
+ verbose => $opt_verbose,
);
-if (eval { $restore->restore }){
+try {
+ $restore->restore;
warn "Restore complete.\n" if $opt_verbose;
exit 0;
-} else {
- chomp $@;
- warn "Error doing restore: $@\n";
+} catch {
+ if (ref $_ and ref $_ eq 'ARRAY') {
+ warn "Restore complete.\n" if $opt_verbose;
+ warn "\n*** Errors encountered doing restore ***\n" . join('', @$_) . "\n";
+ }
+ else {
+ chomp $_;
+ warn "Error doing restore: $_\n";
+ }
exit 1;
-}
+};
sub usage {
diff --git a/lib/Brackup/Restore.pm b/lib/Brackup/Restore.pm
index 8151e41..cb17983 100644
--- a/lib/Brackup/Restore.pm
+++ b/lib/Brackup/Restore.pm
@@ -7,6 +7,7 @@ use POSIX qw(mkfifo);
use Fcntl qw(O_RDONLY O_CREAT O_WRONLY O_TRUNC);
use String::Escape qw(unprintable);
use File::stat;
+use Try::Tiny;
use Brackup::DecryptedFile;
use Brackup::Decrypt;
@@ -14,12 +15,13 @@ sub new {
my ($class, %opts) = @_;
my $self = bless {}, $class;
- $self->{to} = delete $opts{to}; # directory we're restoring to
- $self->{prefix} = delete $opts{prefix}; # directory/file filename prefix, or "" for all
- $self->{filename}= delete $opts{file}; # filename we're restoring from
- $self->{config} = delete $opts{config}; # brackup config (if available)
- $self->{conflict} = delete $opts{conflict};
- $self->{verbose} = delete $opts{verbose};
+ $self->{to} = delete $opts{to}; # directory we're restoring to
+ $self->{prefix} = delete $opts{prefix}; # directory/file filename prefix, or "" for all
+ $self->{filename} = delete $opts{file}; # filename we're restoring from
+ $self->{config} = delete $opts{config}; # brackup config (if available)
+ $self->{onerror} = delete $opts{onerror} || 'abort';
+ $self->{conflict} = delete $opts{conflict} || 'abort';
+ $self->{verbose} = delete $opts{verbose};
$self->{_local_uid_map} = {}; # remote/metafile uid -> local uid
$self->{_local_gid_map} = {}; # remote/metafile gid -> local gid
@@ -98,6 +100,7 @@ sub restore {
}
@files = sort { $a->{fst_dig} cmp $b->{fst_dig} } @files;
+ my @errors;
my $restore_count = 0;
for my $it (@dirs, @files, @rest) {
my $type = $it->{Type} || "f";
@@ -131,12 +134,18 @@ sub restore {
$it->{GID} ||= $meta->{DefaultGID};
warn " * restoring $path_escaped to $full_escaped\n" if $self->{verbose};
- $self->_restore_link ($full, $it) if $type eq "l";
- $self->_restore_directory($full, $it) if $type eq "d";
- $self->_restore_fifo ($full, $it) if $type eq "p";
- $self->_restore_file ($full, $it) if $type eq "f";
-
- $self->_chown($full, $it, $type, $meta) if $it->{UID} || $it->{GID};
+ try {
+ $self->_restore_link ($full, $it) if $type eq "l";
+ $self->_restore_directory($full, $it) if $type eq "d";
+ $self->_restore_fifo ($full, $it) if $type eq "p";
+ $self->_restore_file ($full, $it) if $type eq "f";
+
+ $self->_chown($full, $it, $type, $meta) if $it->{UID} || $it->{GID};
+
+ } catch {
+ die $_ unless $self->{onerror} eq 'continue';
+ push @errors, $_;
+ };
}
# clear chunk cached by _restore_file
@@ -147,6 +156,7 @@ sub restore {
warn " * fixing stat info\n" if $self->{verbose};
$self->_exec_statinfo_updates;
warn " * done\n" if $self->{verbose};
+ die \@errors if @errors;
return 1;
} else {
die "nothing found matching '$self->{prefix}'.\n" if $self->{prefix};
@@ -269,7 +279,7 @@ sub _restore_directory {
my ($self, $full, $it) = @_;
# Apply conflict checks to directories
- if (-d $full && $self->{conflict}) {
+ if (-d $full && $self->{conflict} ne 'abort') {
return if $self->_can_skip($full, $it);
}
@@ -286,7 +296,7 @@ sub _restore_link {
if (-e $full) {
die "Link $full ($it->{Path}) already exists. Aborting."
- unless $self->{conflict};
+ if $self->{conflict} eq 'abort';
return if $self->_can_skip($full, $it);
# Can't overwrite symlinks, so unlink explicitly if we're not skipping
@@ -304,7 +314,7 @@ sub _restore_fifo {
if (-e $full) {
die "Named pipe/fifo $full ($it->{Path}) already exists. Aborting."
- unless $self->{conflict};
+ if $self->{conflict} eq 'abort';
return if $self->_can_skip($full, $it);
# Can't overwrite fifos, so unlink explicitly if we're not skipping
@@ -322,7 +332,7 @@ sub _restore_file {
if (-e $full && -s $full) {
die "File $full ($it->{Path}) already exists. Aborting."
- unless $self->{conflict};
+ if $self->{conflict} eq 'abort';
return if $self->_can_skip($full, $it);
}
# If $full exists, unlink (in case readonly when overwriting would fail)
@@ -386,9 +396,8 @@ sub _restore_file {
$sha1->addfile($readfh);
my $actual_dig = $sha1->hexdigest;
- # TODO: support --onerror={continue,prompt}, etc, but for now we just die
unless ($actual_dig eq $good_dig || $full =~ m!\.brackup-digest\.db\b!) {
- die "Digest of restored file ($full) doesn't match";
+ die "Digest of restored file ($full) doesn't match:\n Got: $actual_dig\n Expected: $good_dig\n";
}
}
commit 1e39d8489458c616c04c42fff13177213289cfa1
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Thu Mar 8 05:00:53 2012 +0000
Add Try::Tiny requirement to Makefile.PL.
git-svn-id: http://brackup.googlecode.com/svn/trunk@354 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/Makefile.PL b/Makefile.PL
index e8ad1cb..6896279 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -8,13 +8,14 @@ WriteMakefile( NAME => 'Brackup',
EXE_FILES => [ 'brackup', 'brackup-restore', 'brackup-target',
'brackup-mount', 'brackup-verify-inventory', ],
PREREQ_PM => {
- 'DBD::SQLite' => 0,
- 'Digest::SHA1' => 0,
- 'DBI' => 0,
- 'String::Escape' => 0,
- 'IO::InnerFile' => 0,
- 'File::Temp' => 0.19, # require a seekable File::Temp + EXLOCK support
'ExtUtils::Manifest' => 1.52, # For spaces in files in MANIFEST
+ 'DBD::SQLite' => 0,
+ 'Digest::SHA1' => 0,
+ 'DBI' => 0,
+ 'File::Temp' => 0.19, # require a seekable File::Temp + EXLOCK support
+ 'IO::InnerFile' => 0,
+ 'String::Escape' => 0,
+ 'Try::Tiny' => 0,
'Test::More' => 0.88, # For done_testing
},
ABSTRACT_FROM => 'lib/Brackup.pm',
commit 71a4c948e05e386e4c78133f7d88cd0dc6e2b156
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Thu Mar 8 05:01:15 2012 +0000
Add onerror => continue and webhook entries to Changes.
git-svn-id: http://brackup.googlecode.com/svn/trunk@355 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/Changes b/Changes
index a43998e..c072341 100644
--- a/Changes
+++ b/Changes
@@ -1,6 +1,11 @@
- - add --conflict [skip|overwrite|update] options to brackup-restore (Gavin
- Carr)
+ - gavincarr: add simple webhook support (Root webhook_url setting),
+ firing on backup success with BackupStats data
+
+ - gavincarr: add --onerror [abort|continue] option to brackup-restore
+
+ - gavincarr: add --conflict [abort|skip|overwrite|update] option to
+ brackup-restore
1.10 (2010-10-31)
commit f97291d40106e2de8c7bc0e6680a4b2a604524c7
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Fri Mar 9 03:25:05 2012 +0000
Add brackup-verify-chunks utility.
git-svn-id: http://brackup.googlecode.com/svn/trunk@356 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/MANIFEST b/MANIFEST
index 3ccd30d..82a47be 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -9,6 +9,7 @@ brackup-mount
brackup-restore
brackup-target
brackup-verify-inventory
+brackup-verify-chunks
doc/data-structures.txt
doc/databases.txt
doc/design-decisions.txt
diff --git a/Makefile.PL b/Makefile.PL
index 6896279..31071f3 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -6,7 +6,9 @@ use ExtUtils::MakeMaker;
WriteMakefile( NAME => 'Brackup',
VERSION_FROM => 'lib/Brackup.pm',
EXE_FILES => [ 'brackup', 'brackup-restore', 'brackup-target',
- 'brackup-mount', 'brackup-verify-inventory', ],
+ 'brackup-mount', 'brackup-verify-inventory',
+ 'brackup-verify-chunks',
+ ],
PREREQ_PM => {
'ExtUtils::Manifest' => 1.52, # For spaces in files in MANIFEST
'DBD::SQLite' => 0,
diff --git a/brackup-verify-chunks b/brackup-verify-chunks
new file mode 100755
index 0000000..938a560
--- /dev/null
+++ b/brackup-verify-chunks
@@ -0,0 +1,156 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+brackup-verify-chunks - utility to check brackup chunk consistency on a
+brackup target, reporting on errors (and optionally deleting bad chunks)
+
+=head1 SYNOPSIS
+
+ brackup-verify-chunks [-v] [-q] [--delete] [--limit=<count>] [<brackup_dir>]
+
+=head2 ARGUMENTS
+
+=over 4
+
+=item <brackup_dir>
+
+Brackup directory root to check. Default: current directory.
+
+=back
+
+=head2 OPTIONS
+
+=over 4
+
+=item --delete
+
+Optional. Delete invalid chunks from the tree.
+
+=item --limit|-l=<count>
+
+Optional. Delete no more than this many chunks (i.e. a safety check).
+
+=item --verbose|-v
+
+Optional. Give more verbose output.
+
+=item --quiet|-q
+
+Optional. Give no output except for errors.
+
+=back
+
+=head1 SEE ALSO
+
+L<brackup>
+
+L<Brackup::Manual::Overview>
+
+=head1 AUTHOR
+
+Gavin Carr <gavin at openfusion.com.au>
+
+Copyright (c) 2008-2012 Gavin Carr.
+
+This module is free software. You may use, modify, and/or redistribute this
+software under the terms of same terms as perl itself.
+
+=cut
+
+use strict;
+use warnings;
+
+use Cwd;
+use Getopt::Long;
+use File::Find::Rule;
+use Digest::SHA1 qw(sha1_hex);
+
+$|=1;
+
+sub usage { die "brackup-verify-chunks [-q|-v] [--delete] [--limit=<count>] [<brackup_dir>]\nbrackup-verify-chunks --help\n" }
+
+my ($opt_help, $opt_quiet, $opt_verbose, $opt_delete, $opt_limit);
+usage() unless
+ GetOptions(
+ 'delete' => \$opt_delete,
+ 'limit|l=i' => \$opt_limit,
+ 'verbose|v+' => \$opt_verbose,
+ 'quiet|q' => \$opt_quiet,
+ 'help|h|?' => \$opt_help,
+ );
+
+if ($opt_help) {
+ eval "use Pod::Usage;";
+ Pod::Usage::pod2usage( -verbose => 1, -exitval => 0 );
+ exit 0;
+}
+usage() unless @ARGV < 2;
+
+my $dir = shift @ARGV || cwd;
+chdir $dir or die "Failed to chdir to $dir: $!\n";
+
+if ($opt_limit && ! $opt_delete) {
+ warn "--limit is only used with --delete - ignoring\n";
+}
+my $deleting = $opt_delete ? ' - deleting' : '';
+
+my $total = 0;
+if ($opt_verbose) {
+ print "Counting chunks ... ";
+ File::Find::Rule->name('*.chunk')->file()->exec(sub { $total++ })->in($dir);
+ print "found $total\n";
+ $total /= 100;
+}
+
+my ($count, $ok, $bad, $deleted) = (0, 0, 0, 0);
+File::Find::Rule->name('*.chunk')
+ ->file()
+ ->exec( sub {
+ my ($shortname, $path, $fullname) = @_;
+ $path =~ s!^$dir/?!!;
+
+ open my $fh, '<', $fullname
+ or die "Cannot open '$fullname': $!";
+ my $text = '';
+ {
+ local $/;
+ $text = <$fh>;
+ }
+ close $fh;
+
+ my $calc_sum = sha1_hex($text);
+ my ($name_sum) = $shortname =~ m/^sha1[:\.]([0-9A-F]+)/i;
+
+ if ($opt_verbose && $opt_verbose == 1) {
+ printf "Checked %s chunks (%0.1f%%)\n", $count, $count / $total
+ if $count && $count % 100 == 0;
+ }
+ elsif ($opt_verbose && $opt_verbose >= 2) {
+ printf "Checking %s, checksum %s (%0.01f%%)\n",
+ "$path/$shortname", $name_sum eq $calc_sum ? 'ok' : $calc_sum,
+ $count / $total;
+ }
+
+ if ($name_sum ne $calc_sum) {
+ warn "Error: chunk $path/$shortname\n has invalid checksum ${calc_sum}$deleting\n";
+ $bad++;
+ if ($opt_delete) {
+ if ($opt_limit && $deleted >= $opt_limit) {
+ die "Delete limit exceeded - check terminating\n";
+ }
+ unlink $fullname;
+ $deleted++;
+ }
+ } else {
+ $ok++;
+ }
+ $count++;
+ })
+ ->in($dir);
+
+print "Checked $count chunks: $ok good, $bad bad.\n" unless $opt_quiet;
+exit $bad ? 1 : 0;
+
+# vim:sw=4
+
commit 239dd952dc4cf32d1671bc5c620ba20dde1b3837
Author: gavin at openfusion.com.au <gavin at openfusion.com.au@c3152786-d43a-0410-8499-09eb43983aed>
Date: Fri Mar 9 04:16:48 2012 +0000
Allow brackup -v -v, and some extra debug messages to Backup::backup.
git-svn-id: http://brackup.googlecode.com/svn/trunk@357 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/brackup b/brackup
index ae8fe2d..4fdceb6 100755
--- a/brackup
+++ b/brackup
@@ -121,7 +121,7 @@ usage() unless
GetOptions(
'from=s' => \$src_name,
'to=s' => \$target_name,
- 'verbose' => \$opt_verbose,
+ 'verbose+' => \$opt_verbose,
'zenity-progress' => \$opt_zenityprogress,
'output=s' => \$backup_file,
'save-stats:s' => \$stats_file,
@@ -218,6 +218,7 @@ if (my $stats = eval { $backup->backup($backup_file) }) {
if ($stats_file) {
$stats->print($stats_file);
}
+ warn "Stats complete.\n" if $opt_verbose;
exit 0;
} else {
warn "Error running backup: $@\n";
diff --git a/lib/Brackup/Backup.pm b/lib/Brackup/Backup.pm
index fd9b5c3..1127b71 100644
--- a/lib/Brackup/Backup.pm
+++ b/lib/Brackup/Backup.pm
@@ -219,6 +219,7 @@ sub backup {
# encrypt it
if (@gpg_rcpts) {
+ $self->debug_more(" * encrypting ... \n");
$schunk->set_encrypted_chunkref($gpg_pm->enc_chunkref_of($pchunk));
}
@@ -241,12 +242,15 @@ sub backup {
$self->flush_files($metafh);
}
$comp_chunk ||= Brackup::CompositeChunk->new($root, $target);
+ $self->debug_more(" * appending to composite chunk ... \n");
$comp_chunk->append_little_chunk($schunk);
} else {
# store it regularly, as its own chunk on the target
+ $self->debug_more(" * storing ... \n");
$target->store_chunk($schunk)
or die "Chunk storage failed.\n";
$target->add_to_inventory($pchunk => $schunk);
+ $self->debug_more(" * chunk stored\n");
}
# if only this worked... (LWP protocol handler seems to
@@ -471,6 +475,13 @@ sub debug {
print $line, "\n";
}
+sub debug_more {
+ my $self = shift;
+ return unless $self->{verbose} && $self->{verbose} >= 2;
+ $self->report_open_files;
+ $self->debug(@_);
+}
+
sub report_progress {
my ($self, $percent, $message) = @_;
commit ba52080b7ab1cfba3b21baca35b495bc805c03e8
Author: cervlean at gmail.com <cervlean at gmail.com@c3152786-d43a-0410-8499-09eb43983aed>
Date: Mon Sep 24 23:31:40 2012 +0000
Silence undef warning in Brackup::Decrypt.
git-svn-id: http://brackup.googlecode.com/svn/trunk@358 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/Decrypt.pm b/lib/Brackup/Decrypt.pm
index 8bfa614..ccd9027 100644
--- a/lib/Brackup/Decrypt.pm
+++ b/lib/Brackup/Decrypt.pm
@@ -39,7 +39,7 @@ sub decrypt_file_if_needed {
my ($filename) = @_;
my $meta = slurp($filename, decompress => 1);
- if ($meta =~ /[\x00-\x08]/) { # silly is-binary heuristic
+ if ($meta and $meta =~ /[\x00-\x08]/) { # silly is-binary heuristic
my $new_file = decrypt_file($filename,no_batch => 1);
if (defined $new_file) {
warn "Decrypted ${filename} to ${new_file}.\n";
commit 4a22795936676eb3a20ba4b209be74b196d77d50
Author: cervlean at gmail.com <cervlean at gmail.com@c3152786-d43a-0410-8499-09eb43983aed>
Date: Tue Oct 2 23:29:57 2012 +0000
Fix typo in brackup-verify-inventory, clarify low chunk count error.
git-svn-id: http://brackup.googlecode.com/svn/trunk@359 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/brackup-verify-inventory b/brackup-verify-inventory
index 98b93ac..a34a222 100755
--- a/brackup-verify-inventory
+++ b/brackup-verify-inventory
@@ -115,9 +115,9 @@ print "Fetching list of chunks from target\n" if $opt_verbose;
my %chunks = map { $_ => 1 } $target->chunks;
my $chunk_count = scalar keys %chunks;
-# Sanity check if in delete mode - abort if chunk mismatch > 10%
-die sprintf("Error: low target chunk count (%d chunks < 90% of %d inventory entries) - aborting\n",
- $chunk_count, $inv_db->count)
+# Sanity check if in delete mode - abort if target chunk count < 90% of inventory
+die sprintf("Error: target has only %d chunks (%.1f%%) of expected %d chunks in inventory - aborting\n",
+ $chunk_count, $chunk_count * 100 / $inv_db->count, $inv_db->count)
if $opt_delete && $chunk_count < 0.9 * $inv_db->count;
print "Checking inventory entries\n" if $opt_verbose;
commit aa14efd1806800b2b4e22713d0625f04936718fa
Author: cervlean at gmail.com <cervlean at gmail.com@c3152786-d43a-0410-8499-09eb43983aed>
Date: Wed Oct 3 23:35:18 2012 +0000
Add --force option to brackup-verify-inventory.
git-svn-id: http://brackup.googlecode.com/svn/trunk@360 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/brackup-verify-inventory b/brackup-verify-inventory
index a34a222..36fa729 100755
--- a/brackup-verify-inventory
+++ b/brackup-verify-inventory
@@ -29,7 +29,14 @@ This must match a [TARGET:NAME] config section in your ~/.brackup.conf.
=item --delete
-Optional. Delete orphaned entries found in the inventory.
+Optional. Delete orphaned entries found in the inventory. If orphaned chunks
+comprise more than 10% of the inventory, brackup-verify-inventory will abort
+with an error. Use L<--force> to force deletion in this case.
+
+=item --force|-f
+
+Force deletion of orphaned entries when they comprise more than 10% of the
+inventory.
=item --verbose|-v
@@ -70,7 +77,7 @@ use Brackup;
$|=1;
-my ($opt_help, $opt_quiet, $opt_verbose, $opt_delete);
+my ($opt_help, $opt_quiet, $opt_verbose, $opt_delete, $opt_force);
my $config_file = Brackup::Config->default_config_file_name;
@@ -80,13 +87,14 @@ sub usage {
$why =~ s/\s+$//;
$why = "Error: $why\n\n";
}
- die "${why}brackup-verify-inventory [-q|-v] [--delete] <target_name>\nbrackup-verify-inventory --help\n";
+ die "${why}brackup-verify-inventory [-q|-v] [--delete] [--force] <target_name>\nbrackup-verify-inventory --help\n";
}
usage() unless
GetOptions(
'config=s' => \$config_file,
'delete' => \$opt_delete,
+ 'force|f' => \$opt_force,
'quiet|q' => \$opt_quiet,
'verbose|v+' => \$opt_verbose,
'help|h|?' => \$opt_help,
@@ -116,9 +124,16 @@ my %chunks = map { $_ => 1 } $target->chunks;
my $chunk_count = scalar keys %chunks;
# Sanity check if in delete mode - abort if target chunk count < 90% of inventory
-die sprintf("Error: target has only %d chunks (%.1f%%) of expected %d chunks in inventory - aborting\n",
- $chunk_count, $chunk_count * 100 / $inv_db->count, $inv_db->count)
- if $opt_delete && $chunk_count < 0.9 * $inv_db->count;
+if ($opt_delete && $chunk_count < 0.9 * $inv_db->count) {
+ if (not $opt_force) {
+ die sprintf("Error: target has only %d chunks (%.1f%%) of expected %d chunks in inventory - aborting\n",
+ $chunk_count, $chunk_count * 100 / $inv_db->count, $inv_db->count);
+ }
+ else {
+ warn sprintf("Warning: target has only %d chunks (%.1f%%) of expected %d chunks in inventory - continuing with --force\n",
+ $chunk_count, $chunk_count * 100 / $inv_db->count, $inv_db->count);
+ }
+}
print "Checking inventory entries\n" if $opt_verbose;
my ($count, $ok, $bad, $skip) = (0, 0, 0, 0);
commit fe748d5c52276fd7f27d23638c8a189b1363aa14
Author: cervlean at gmail.com <cervlean at gmail.com@c3152786-d43a-0410-8499-09eb43983aed>
Date: Thu Oct 4 00:55:18 2012 +0000
Handle unexpected backup names better in Target::prune.
git-svn-id: http://brackup.googlecode.com/svn/trunk@361 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/Target.pm b/lib/Brackup/Target.pm
index 4089b9d..afd53df 100644
--- a/lib/Brackup/Target.pm
+++ b/lib/Brackup/Target.pm
@@ -124,9 +124,13 @@ sub prune {
# select backups to delete
my (%backups, @backups_to_delete) = ();
foreach my $backup_name (map {$_->filename} $self->backups) {
- $backup_name =~ /^(.+)-\d+$/;
- $backups{$1} ||= [];
- push @{ $backups{$1} }, $backup_name;
+ if ($backup_name =~ /^(.+)-\d+$/) {
+ $backups{$1} ||= [];
+ push @{ $backups{$1} }, $backup_name;
+ }
+ else {
+ warn "Unexpected backup name format: '$backup_name' does not match /-d+\$/";
+ }
}
foreach my $source (keys %backups) {
next if $opt{source} && $source ne $opt{source};
commit 4b5ef651e996d72f8854cddacab1a73b8e1db1b9
Author: cervlean at gmail.com <cervlean at gmail.com@c3152786-d43a-0410-8499-09eb43983aed>
Date: Thu Oct 4 01:07:52 2012 +0000
Add -n alias for --dry-run to brackup/brackup-target.
git-svn-id: http://brackup.googlecode.com/svn/trunk@362 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/brackup b/brackup
index 4fdceb6..0013dcd 100755
--- a/brackup
+++ b/brackup
@@ -43,7 +43,7 @@ file. If =FILE is omitted, defaults to "source-target-YYYYMMDD.stats."
Show status during backup.
-=item --dry-run
+=item --dry-run|-n
Don't actually store any data on the target.
@@ -126,7 +126,7 @@ usage() unless
'output=s' => \$backup_file,
'save-stats:s' => \$stats_file,
'help' => \$opt_help,
- 'dry-run' => \$opt_dryrun,
+ 'dry-run|n' => \$opt_dryrun,
'du-stats' => \$opt_du_stats,
'config=s' => \$config_file,
'list-sources' => \$opt_list_sources,
diff --git a/brackup-target b/brackup-target
index 91f2817..c4cfe4d 100755
--- a/brackup-target
+++ b/brackup-target
@@ -25,7 +25,7 @@ Destination to write files to. Defaults to current working directory.
Be verbose with status.
-=item --dry-run
+=item --dry-run|-n
Do not actually execute write operations.
@@ -87,14 +87,14 @@ my $opt_interactive;
my $opt_source;
usage() unless
GetOptions(
- 'verbose+' => \$opt_verbose,
- 'dest=s' => \$destdir,
- 'config=s' => \$config_file,
+ 'verbose+' => \$opt_verbose,
+ 'dest=s' => \$destdir,
+ 'config=s' => \$config_file,
'keep-backups=i' => \$opt_keep_backups,
- 'dry-run' => \$opt_dryrun,
- 'interactive' => \$opt_interactive,
- 'source=s' => \$opt_source,
- 'help' => \$opt_help,
+ 'dry-run|n' => \$opt_dryrun,
+ 'interactive' => \$opt_interactive,
+ 'source=s' => \$opt_source,
+ 'help' => \$opt_help,
);
if ($destdir) {
commit a594c69273c21801edd23597fe6ee4e0bd610805
Author: cervlean at gmail.com <cervlean at gmail.com@c3152786-d43a-0410-8499-09eb43983aed>
Date: Tue Oct 9 03:43:19 2012 +0000
Move Root foreach_file backup-digest check out of pattern loop.
git-svn-id: http://brackup.googlecode.com/svn/trunk@363 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/Root.pm b/lib/Brackup/Root.pm
index 04f0751..f641455 100644
--- a/lib/Brackup/Root.pm
+++ b/lib/Brackup/Root.pm
@@ -120,6 +120,8 @@ sub foreach_file {
# the backup meta files later, so let's skip it.
next if $self->{digcache_file} && $path eq $self->{digcache_file};
+ next if $path =~ m!(^|/)\.brackup-digest\.db(-journal)?$!;
+
# GC: seems to work fine as of at least gpg 1.4.5, so commenting out
# gpg seems to barf on files ending in whitespace, blowing
# stuff up, so we just skip them instead...
@@ -134,7 +136,6 @@ sub foreach_file {
foreach my $pattern (@{ $self->{ignore} }) {
next DENTRY if $path =~ /$pattern/;
next DENTRY if $is_dir && "$path/" =~ /$pattern/;
- next DENTRY if $path =~ m!(^|/)\.brackup-digest\.db(-journal)?$!;
}
$statcache{$path} = $statobj;
commit 48a67fa7de150114b11e3369527e0b64617fced4
Author: cervlean at gmail.com <cervlean at gmail.com@c3152786-d43a-0410-8499-09eb43983aed>
Date: Tue Oct 9 03:43:52 2012 +0000
Add /etc/brackup/brackup.conf as a unix alternative to ~/.brackup.conf.
git-svn-id: http://brackup.googlecode.com/svn/trunk@364 c3152786-d43a-0410-8499-09eb43983aed
diff --git a/lib/Brackup/Config.pm b/lib/Brackup/Config.pm
index 665bb79..3e05967 100644
--- a/lib/Brackup/Config.pm
+++ b/lib/Brackup/Config.pm
@@ -29,7 +29,7 @@ sub load {
open (my $fh, $file) or do {
if (write_dummy_config($file)) {
- die "Your config file needs tweaking. I put a commented-out template at: $file\n";
+ die "No config file found. I've created a commented-out template at: $file\n";
} else {
die "No config file at: $file\n";
}
@@ -89,7 +89,12 @@ sub default_config_file_name {
if ($ENV{HOME}) {
# Default for UNIX folk
- return "$ENV{HOME}/.brackup.conf";
+ my $home_brackup_conf = "$ENV{HOME}/.brackup.conf";
+ return $home_brackup_conf if -f $home_brackup_conf;
+ my $etc_brackup_conf = "/etc/brackup/brackup.conf";
+ return $etc_brackup_conf if -f $etc_brackup_conf;
+ # Fall back to ~/.brackup.conf if none exist
+ return $home_brackup_conf;
}
elsif ($ENV{APPDATA}) {
# For Windows users
diff --git a/lib/Brackup/Manual/Overview.pod b/lib/Brackup/Manual/Overview.pod
index dd4c763..aaf58c6 100644
--- a/lib/Brackup/Manual/Overview.pod
+++ b/lib/Brackup/Manual/Overview.pod
@@ -10,8 +10,7 @@ Run B<brackup> to initialize your config file. You'll see:
$ brackup
Error:
- Your config file needs tweaking.
- I put a commented-out template at: /home/bradfitz/.brackup.conf
+ No config file found. I've created a commented-out template at: $HOME/.brackup.conf
brackup --from=[source_name] --to=[target_name] [--output=<backup_metafile.brackup>]
brackup --help
@@ -25,6 +24,10 @@ Tweak as appropriate.
For details on what's tweakable, see L<Brackup::Root> (a "source"), or
L<Brackup::Target> (a destination).
+If you're running on a unix machine and are setting up system-wide
+backups, you might want to move your config file to
+C</etc/brackup/brackup.conf>.
+
=head2 Do a backup
Now that you've got a source and target named, run a backup. I like
commit 7e3686e129be3a4c2e7b7a1c54a62ad9e41e6a5c
Merge: 98e8632 48a67fa
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Thu Jan 15 16:32:20 2015 -0500
Merge remote branch 'trunk'
Conflicts:
brackup-restore
lib/Brackup/Restore.pm
diff --cc brackup-restore
index 44e6e8b,7adb8ef..18dcd30
--- a/brackup-restore
+++ b/brackup-restore
@@@ -42,16 -40,22 +40,24 @@@ Restore just the directory named (and a
Restore just the file named.
- =item --conflict=skip|overwrite|update|identical
+ =item --onerror=abort|continue
+
+ How to handle restore errors. 'abort' reports and stops as soon as an
+ error is detected. 'continue' continues with the restore, collecting all
+ errors and reporting them at the end of the restore.
+
+ Default: abort.
+
-=item --conflict=abort|skip|overwrite|update
++=item --conflict=abort|skip|overwrite|update|identical
How to handle files that already exist (with size > zero bytes).
'skip' means don't restore, keeping the existing file. 'overwrite'
means always restore, replacing the existing file. 'update' means
-overwrite iff the file we are restoring is newer than the existing one.
+overwrite iff the file we are restoring is newer than the existing
+one. 'identical' means skip files iff their hash matches the expected
+value.
- Default is to abort the restore if a file already exists.
+ Default: abort.
=item --config=NAME
@@@ -115,8 -123,10 +125,10 @@@ usage("Backup metafile '$meta_file' doe
usage("Backup metafile '$meta_file' isn't a file") unless -f $meta_file;
usage("Restore directory '$restore_dir' isn't a directory") if -e $restore_dir && ! -d $restore_dir;
usage("Config file '$config_file' doesn't exist") if $config_file && ! -f $config_file;
- usage("Invalid --conflict option '$conflict' (not skip|overwrite|update)")
- if $conflict && $conflict !~ m/^(skip|overwrite|update)$/;
+ usage("Invalid --onerror option '$onerror' (not abort|continue)")
+ if $onerror !~ m/^(abort|continue)$/;
-usage("Invalid --conflict option '$conflict' (not abort|skip|overwrite|update)")
- if $conflict !~ m/^(abort|skip|overwrite|update)$/;
++usage("Invalid --conflict option '$conflict' (not abort|skip|overwrite|update|identical)")
++ if $conflict !~ m/^(abort|skip|overwrite|update|identical)$/;
$prefix ||= ""; # with -all, "", which means everything
if (! -e $restore_dir) {
diff --cc lib/Brackup/Restore.pm
index 17a8357,cb17983..421c95a
--- a/lib/Brackup/Restore.pm
+++ b/lib/Brackup/Restore.pm
@@@ -7,9 -7,9 +7,10 @@@ use POSIX qw(mkfifo)
use Fcntl qw(O_RDONLY O_CREAT O_WRONLY O_TRUNC);
use String::Escape qw(unprintable);
use File::stat;
+ use Try::Tiny;
use Brackup::DecryptedFile;
use Brackup::Decrypt;
+use Brackup::Util qw(io_sha1);
sub new {
my ($class, %opts) = @_;
commit 7da6e5c5a44a6dcbc108e24664eab5e7826ff654
Author: Alex Vandiver <alexmv at bestpractical.com>
Date: Thu Jan 15 16:39:43 2015 -0500
Perl 5.18 introduces hash key ordering; sort them if we are to compare them
diff --git a/lib/Brackup/Test.pm b/lib/Brackup/Test.pm
index 8329ab0..67a1046 100644
--- a/lib/Brackup/Test.pm
+++ b/lib/Brackup/Test.pm
@@ -178,6 +178,7 @@ sub ok_dirs_match {
if ($has_diff) {
use Data::Dumper;
+ $Data::Dumper::Sortkeys = 1;
my $pre_dump = Dumper($pre_ls);
my $post_dump = Dumper($post_ls);
my $diff = Text::Diff::diff(\$pre_dump, \$post_dump);
@@ -195,6 +196,7 @@ sub ok_files_match {
if ($has_diff) {
use Data::Dumper;
+ $Data::Dumper::Sortkeys = 1;
my $pre_dump = Dumper($pre_ls);
my $post_dump = Dumper($post_ls);
my $diff = Text::Diff::diff(\$pre_dump, \$post_dump);
-----------------------------------------------------------------------
More information about the Bps-public-commit
mailing list