[Rt-commit] r6176 - in Mnemonic: . lib lib/Mnemonic
lib/Mnemonic/Backend
jesse at bestpractical.com
jesse at bestpractical.com
Mon Oct 9 09:46:54 EDT 2006
Author: jesse
Date: Mon Oct 9 09:46:52 2006
New Revision: 6176
Modified:
Mnemonic/ (props changed)
Mnemonic/bin/mnemonic
Mnemonic/lib/Mnemonic.pm
Mnemonic/lib/Mnemonic/Backend/Filesystem.pm
Mnemonic/lib/Mnemonic/Backend/S3.pm
Mnemonic/lib/Mnemonic/FileSet.pm
Log:
r28186 at 251: jesse | 2006-10-09 09:46:53 -0400
* updates to get it to run for real
Modified: Mnemonic/bin/mnemonic
==============================================================================
--- Mnemonic/bin/mnemonic (original)
+++ Mnemonic/bin/mnemonic Mon Oct 9 09:46:52 2006
@@ -35,7 +35,7 @@
$b->remove_key_from_store($argv{'delete_key'});
}
if ($argv{backup}) {
- my @items = $b->list_backups();
+ my @items;# = $b->list_backups();
my $manifest = $b->backup(
%argv,
stored_keys => \@items
Modified: Mnemonic/lib/Mnemonic.pm
==============================================================================
--- Mnemonic/lib/Mnemonic.pm (original)
+++ Mnemonic/lib/Mnemonic.pm Mon Oct 9 09:46:52 2006
@@ -16,14 +16,15 @@
use base qw/Class::Accessor/;
-BEGIN { __PACKAGE__->mk_accessors(qw/backend config_file config pgp stash/); }
+BEGIN { __PACKAGE__->mk_accessors(qw/backend config_file config pgp stash errors/); }
-our $CHUNKSIZE = 1024000;
+our $CHUNKSIZE = 1024000 * 32 ;
sub init {
my $self = shift;
$self->load_config();
+ $self->errors([]);
$self->pgp(
Mnemonic::Crypto::OpenPGP->new( { config => $self->config } ) );
my $backend = $self->config->{'backend'}
@@ -72,6 +73,7 @@
my $seen_checksum = {};
$seen_checksum->{$_} = "prestored" for ( @{ $args{'stored_keys'} } );
my $on_match = sub {
+ warn "IT IS A SYMLINK: $File::Find::name" if -l $File::Find::name;
my $checksum = $self->store_item(
manifest => $manifest,
filename => $File::Find::name,
@@ -90,14 +92,21 @@
my $hostname = `hostname`;
chomp $hostname;
- my $manifest_id = $self->pgp->key_id
- . '/MANIFEST/'
- . $hostname . "/"
- . time() . '-'
- . $$;
+ my @manifest_parts = ($self->pgp->key_id, 'MANIFEST', $hostname , time() . '-' . $$);
+ my $manifest_id = join('/', @manifest_parts);
+
my $manifest_yaml = YAML::Syck::Dump($manifest);
- $self->backend->store(
- $manifest_id => $self->pgp->encrypt($manifest_yaml) );
+ eval { $self->backend->store( $manifest_id => $self->pgp->encrypt($manifest_yaml) )};
+ if (my $error = $@) {
+ $self->log_error( {manifest => $manifest_id, error => $error } );
+ warn "Failed to store manifest for this backup. All data has been stored, but recovery will be impossible without the MANIFEST. ($error)\n"
+ }
+
+ my $manifest_file = File::Spec->catpath('tmp',join(';', at manifest_parts));
+ YAML::Syck::DumpFile($manifest_file, $manifest);
+ warn "Manifest stored in $manifest_file.\n";
+
+
warn "Completed backup $manifest_id\n";
return $manifest_id;
}
@@ -131,14 +140,16 @@
}
- if ( -d $filename ) {
+ if ( -l $filename ) {
$manifest->{$filename} = {
- type => 'directory',
- statinfo => [ stat($filename) ],
+ type => 'symlink',
+ statinfo => [ lstat($filename) ],
+ target => eval {readlink $filename },
stored => '0'
}
- } elsif ( -f $filename ) {
+ }
+ elsif ( -f $filename ) {
if ( -z $filename ) {
$manifest->{$filename} = {
type => 'file',
@@ -211,7 +222,7 @@
warn "\t= $filename already stored\n";
} else {
$self->encrypt_and_store(
- $checksum => $chunk_ref );
+ $checksum => $chunk_ref, filename => $filename );
my $str = "\t+ $filename ";
$str .= "(from " . $start_at . ") "
@@ -245,6 +256,14 @@
}
}
}
+ elsif ( -d $filename ) {
+ $manifest->{$filename} = {
+ type => 'directory',
+ statinfo => [ stat($filename) ],
+ stored => '0'
+ }
+
+ }
}
sub restore {
@@ -281,7 +300,8 @@
$self->restore_file( $file => $target => $manifest->{$file} );
} elsif ( $manifest->{$file}->{type} eq 'directory' ) {
$self->restore_directory( $file => $target );
-
+ } elsif ( $manifest->{$file}->{type} eq 'symlink' ) {
+ symlink($manifest->{$file}->{'target'} => $target)||die "$target: $!";
} else {
warn "WTF $file" . YAML::Syck::Dump( $manifest->{$file} );
}
@@ -295,7 +315,12 @@
my $dir = shift;
my $target = shift;
- File::Path::mkpath( [$target] );
+ eval { File::Path::mkpath( [$target] ) };
+ if ( my $error = $@ ) {
+ $self->log_error( { restore_directory => $target, error => $error } );
+ warn "!!!! Failed to mkdir $target: $error";
+ }
+
}
sub restore_file {
@@ -304,34 +329,43 @@
my $target = shift;
my $manifest_entry = shift;
- eval {
- if ( ( $manifest_entry->{stored} || 0 ) > 0 ) {
+ eval {
+ if ( ( $manifest_entry->{stored} || 0 ) > 0 )
+ {
- open( my $outfile, ">", $target ) || die $!;
- binmode($outfile);
- foreach my $key ( @{ $manifest_entry->{subkeys} } ) {
- my $file_result = $self->backend->fetch($key);
- my $pt = $self->pgp->decrypt( $file_result->{'value'} );
- warn "\t\trestoring " . $key . "\n";
- print $outfile $pt || die $!;
- }
+ open( my $outfile, ">", $target ) || die $!;
+ binmode($outfile);
+ foreach my $key ( @{ $manifest_entry->{subkeys} } ) {
+ my $file_result = $self->backend->fetch($key);
+ my $pt = $self->pgp->decrypt( $file_result->{'value'} );
+ warn "\t\trestoring " . $key . "\n";
+ print $outfile $pt || die $!;
+ }
- close $outfile || die $!;
+ close $outfile || die $!;
- unless ( $manifest_entry->{sha256} eq $self->_sha256($target) ) {
- warn "$file has an invalid SHA256 sum after decryption";
- }
- } elsif ( $manifest_entry->{size} == 0 ) {
- Shell::Command::touch $target;
+ unless ( $manifest_entry->{sha256} eq $self->_sha256($target) ) {
+ $self->log_error(
+ { restore_file => $file,
+ error => 'invalid checksum',
+ manifest_checksum => $manifest_entry->{sha256},
+ calculated_checksum => $self->_sha256($target)
+ }
+ );
+ warn "$file has an invalid SHA256 sum after decryption";
+ }
+ } elsif ( $manifest_entry->{size} == 0 ) {
+ Shell::Command::touch $target;
- } else {
- warn "Not fetching $file\n";
+ } else {
+ warn "Not fetching $file\n";
+ }
+ };
+ if ( my $error = $@ ) {
+ $self->log_error( { restore_file => $file, error => $error } );
+ warn "!!! FAILED TO RESTORE RESTORE $target: $error";
}
- };
- if ($@) {
- warn "!!! FAILED TO RESTORE RESTORE $target: $@";
- }
}
sub remove_key_from_store {
@@ -416,12 +450,25 @@
my $self = shift;
my $key = shift;
my $plaintext_ref = shift;
-
- my $cyphertext = $self->pgp->encrypt($$plaintext_ref);
- eval { $self->backend->store( $key => $cyphertext, ); };
- if ($@) {
+ my %args = (filename => undef,
+ @_);
+ # Try twice if there's an error storing the file
+ for ( 1 .. 2 ) {
+ my $cyphertext = $self->pgp->encrypt($$plaintext_ref);
+ eval { $self->backend->store( $key => $cyphertext, ); };
+ last unless (my $error =$@);
+ $self->log_error({ $key => $error, %args} );
warn "\t!ERROR: $key: $@\n";
+ sleep 5;
}
}
+
+sub log_error {
+ my $self = shift;
+ my $error = shift;
+ push @{$self->errors()}, $error;
+
+}
+
1;
Modified: Mnemonic/lib/Mnemonic/Backend/Filesystem.pm
==============================================================================
--- Mnemonic/lib/Mnemonic/Backend/Filesystem.pm (original)
+++ Mnemonic/lib/Mnemonic/Backend/Filesystem.pm Mon Oct 9 09:46:52 2006
@@ -48,9 +48,9 @@
File::Path::mkpath( [$indir
] );
- open (my $outfile, ">", $path) || die "Can't open $path ". $!;
- print $outfile $_[0] || die $!;
- close $outfile || die $!;
+ open (my $outfile, ">", $path) || die "Can't open $path: ". $!;
+ print $outfile $_[0] || die "Failed to write to $path: ". $!;
+ close $outfile || die "Failed to close $path: ".$!;
}
sub fetch {
Modified: Mnemonic/lib/Mnemonic/Backend/S3.pm
==============================================================================
--- Mnemonic/lib/Mnemonic/Backend/S3.pm (original)
+++ Mnemonic/lib/Mnemonic/Backend/S3.pm Mon Oct 9 09:46:52 2006
@@ -65,7 +65,9 @@
sub store {
my $self = shift;
- $self->bucket->add_key( @_);
+ unless ( $self->bucket->add_key( @_) ) {
+ die $self->s3->errstr;
+ }
}
Modified: Mnemonic/lib/Mnemonic/FileSet.pm
==============================================================================
--- Mnemonic/lib/Mnemonic/FileSet.pm (original)
+++ Mnemonic/lib/Mnemonic/FileSet.pm Mon Oct 9 09:46:52 2006
@@ -17,7 +17,7 @@
foreach my $entry (@{$self->skip_patterns||[]}) {
$entry =~ s/(\.|\(|\))/\\$1/g;
$entry =~ s/\*/\(\?\:\.\*\)/g;
- $entry = "\/$entry";
+ $entry = "\/$entry" unless $entry =~ /\^|^\//;
push @regexen, $entry;
}
my $skip = join('|', at regexen);
@@ -34,21 +34,25 @@
my $count = 0;
my $skipped = 0;
my $last = 0;
- print "Looking in: " . join(', ', @{$self->search_paths||[]});
+ my $skip = $self->skip_regexes;
+ warn "Looking in: " . join( ', ', @{ $self->search_paths || [] } ) . "\n";
+ warn "Skipping files matching this pattern: $skip\n";
+
+ File::Find::find(
+ { wanted => sub {
+
+ if ( $skip && $File::Find::name =~ $skip ) {
+ warn "\t- $File::Find::name SKIPPED\n";
+ return;
+ }
+
+ &{ $self->on_match() };
+ },
+ follow => 0
+
+ },
- my $skip = $self->skip_regexes;
- File::Find::finddepth(
- sub {
- if( $File::Find::name =~ $skip) {
- warn "\t- $File::Find::name SKIPPED\n";
- return;
- }
-
-
-
- &{ $self->on_match() };
-
- } , @{ $self->search_paths||[] }
+ @{ $self->search_paths || [] }
);
return @files;
}
More information about the Rt-commit
mailing list