[Rt-commit] r6175 - in Mnemonic: . lib lib/Mnemonic
jesse at bestpractical.com
jesse at bestpractical.com
Sat Oct 7 03:01:57 EDT 2006
Author: jesse
Date: Sat Oct 7 03:01:56 2006
New Revision: 6175
Modified:
Mnemonic/ (props changed)
Mnemonic/lib/Mnemonic.pm
Mnemonic/lib/Mnemonic/Backend/Filesystem.pm
Mnemonic/lib/Mnemonic/Backend/S3.pm
Mnemonic/lib/Mnemonic/FileSet.pm
Log:
r28074 at pinglin: jesse | 2006-10-07 03:01:43 -0400
* Better error handling, ability to skip files to back up
Modified: Mnemonic/lib/Mnemonic.pm
==============================================================================
--- Mnemonic/lib/Mnemonic.pm (original)
+++ Mnemonic/lib/Mnemonic.pm Sat Oct 7 03:01:56 2006
@@ -24,8 +24,10 @@
my $self = shift;
$self->load_config();
- $self->pgp( Mnemonic::Crypto::OpenPGP->new( { config => $self->config } ) );
- my $backend = $self->config->{'backend'} || "Mnemonic::Backend::Filesystem";
+ $self->pgp(
+ Mnemonic::Crypto::OpenPGP->new( { config => $self->config } ) );
+ my $backend = $self->config->{'backend'}
+ || "Mnemonic::Backend::Filesystem";
$backend->require() || die $@;
$self->backend( $backend->new( { config => $self->config() } ) );
$self->backend->init();
@@ -150,9 +152,15 @@
eval {
my @statinfo = stat $filename;
- if ( my $record = $self->stash->already_stored( $self->backend, $filename , \@statinfo) )
+ if (my $record = $self->stash->already_stored(
+ $self->backend, $filename, \@statinfo
+ )
+ )
{
- warn "\t. " . $filename . " already stored (cache id " . $record->id . ")\n";
+ warn "\t. "
+ . $filename
+ . " already stored (cache id "
+ . $record->id . ")\n";
$manifest->{$filename} = {
subkeys => $record->subkeys,
statinfo => $record->statinfo,
@@ -162,57 +170,74 @@
using_cached_item => $record->id
};
- } else {
- my $sha256 = $self->_sha256($filename);
- if ( my $record = $self->stash->has_hash_of_file_content($self->backend, $sha256)) {
- warn "\t. " . $filename . " matches already stored content (cache id " . $record->id . ")\n";
- $manifest->{$filename} = {
- subkeys => $record->subkeys,
- statinfo => [stat($filename)],
- stored => 1,
- sha256 => $sha256,
- type => 'file',
- using_cached_content => $record->id
-
- };
- $self->stash->store_file_info( %{ $manifest->{$filename} }, filename => $filename, backend => $self->backend);
- }
-
+ } else {
+ my $sha256 = $self->_sha256($filename);
+ if (my $record = $self->stash->has_hash_of_file_content(
+ $self->backend, $sha256
+ )
+ )
+ {
+ warn "\t. "
+ . $filename
+ . " matches already stored content (cache id "
+ . $record->id . ")\n";
+ $manifest->{$filename} = {
+ subkeys => $record->subkeys,
+ statinfo => [ stat($filename) ],
+ stored => 1,
+ sha256 => $sha256,
+ type => 'file',
+ using_cached_content => $record->id
+
+ };
+ $self->stash->store_file_info(
+ %{ $manifest->{$filename} },
+ filename => $filename,
+ backend => $self->backend
+ );
+ }
- else {
+ else {
- my $len = $statinfo[7];
- my $start_at = 0;
- my @subkeys;
- open( my $fh, "<", $filename ) || die $!;
- while ( $start_at <= $len ) {
- my $chunk_ref = $self->read_chunk( $fh, $CHUNKSIZE );
- my $checksum = $self->_get_checksum($chunk_ref);
- if ( $args{'stored_keys'}->{$checksum} ) {
- warn "\t= $filename already stored\n";
- } else {
- $self->encrypt_and_store(
- $checksum => $chunk_ref );
- warn "\t+ $filename ";
-
- warn "(from " . $start_at . ") " if ($start_ad > 0);
- warn "stored\n";
+ my $len = $statinfo[7];
+ my $start_at = 0;
+ my @subkeys;
+ open( my $fh, "<", $filename ) || die $!;
+ while ( $start_at <= $len ) {
+ my $chunk_ref
+ = $self->read_chunk( $fh, $CHUNKSIZE );
+ my $checksum = $self->_get_checksum($chunk_ref);
+ if ( $args{'stored_keys'}->{$checksum} ) {
+ warn "\t= $filename already stored\n";
+ } else {
+ $self->encrypt_and_store(
+ $checksum => $chunk_ref );
+ my $str = "\t+ $filename ";
+
+ $str .= "(from " . $start_at . ") "
+ if ( $start_at > 0 );
+ $str .= "stored\n";
+ warn $str;
+ }
+ push @subkeys, $checksum;
+ $start_at += $CHUNKSIZE;
}
- push @subkeys, $checksum;
- $start_at += $CHUNKSIZE;
- }
- close($fh);
+ close($fh);
- $manifest->{$filename} = {
- subkeys => \@subkeys,
- statinfo => \@statinfo,
- stored => 1,
- sha256 => $sha256,
- type => 'file'
- };
- $self->stash->store_file_info( %{ $manifest->{$filename} }, filename => $filename, backend => $self->backend);
+ $manifest->{$filename} = {
+ subkeys => \@subkeys,
+ statinfo => \@statinfo,
+ stored => 1,
+ sha256 => $sha256,
+ type => 'file'
+ };
+ $self->stash->store_file_info(
+ %{ $manifest->{$filename} },
+ filename => $filename,
+ backend => $self->backend
+ );
+ }
}
- }
};
if ($@) {
$manifest->{'!errors'}->{$filename} = $@;
@@ -279,12 +304,12 @@
my $target = shift;
my $manifest_entry = shift;
+ eval {
if ( ( $manifest_entry->{stored} || 0 ) > 0 ) {
open( my $outfile, ">", $target ) || die $!;
binmode($outfile);
foreach my $key ( @{ $manifest_entry->{subkeys} } ) {
-
my $file_result = $self->backend->fetch($key);
my $pt = $self->pgp->decrypt( $file_result->{'value'} );
warn "\t\trestoring " . $key . "\n";
@@ -303,6 +328,10 @@
warn "Not fetching $file\n";
}
+ };
+ if ($@) {
+ warn "!!! FAILED TO RESTORE RESTORE $target: $@";
+ }
}
sub remove_key_from_store {
Modified: Mnemonic/lib/Mnemonic/Backend/Filesystem.pm
==============================================================================
--- Mnemonic/lib/Mnemonic/Backend/Filesystem.pm (original)
+++ Mnemonic/lib/Mnemonic/Backend/Filesystem.pm Sat Oct 7 03:01:56 2006
@@ -11,7 +11,7 @@
-__PACKAGE__->mk_accessors(qw/path id/);
+__PACKAGE__->mk_accessors(qw/path id config/);
sub init {
my $self = shift;
Modified: Mnemonic/lib/Mnemonic/Backend/S3.pm
==============================================================================
--- Mnemonic/lib/Mnemonic/Backend/S3.pm (original)
+++ Mnemonic/lib/Mnemonic/Backend/S3.pm Sat Oct 7 03:01:56 2006
@@ -30,7 +30,7 @@
my $hostname = `hostname`;
chomp($hostname);
- my $bucketname = 'bps_s3_client-0.3-' . $hostname;
+ my $bucketname = 'mnemonic.bestpractical.com.0.3.' . $hostname;
my $bucket_list = $self->s3->buckets();
Modified: Mnemonic/lib/Mnemonic/FileSet.pm
==============================================================================
--- Mnemonic/lib/Mnemonic/FileSet.pm (original)
+++ Mnemonic/lib/Mnemonic/FileSet.pm Sat Oct 7 03:01:56 2006
@@ -18,10 +18,10 @@
$entry =~ s/(\.|\(|\))/\\$1/g;
$entry =~ s/\*/\(\?\:\.\*\)/g;
$entry = "\/$entry";
- warn $entry;
- push @regexen, qr/$entry/i;
+ push @regexen, $entry;
}
- $self->skip_regexes(@regexen||[]);
+ my $skip = join('|', at regexen);
+ $self->skip_regexes($skip);
}
@@ -30,15 +30,22 @@
my $self = shift;
$self->init();
my @files;
- my @regexen = @{ $self->skip_regexes };
$|++; # turn off buffering
my $count = 0;
my $skipped = 0;
my $last = 0;
print "Looking in: " . join(', ', @{$self->search_paths||[]});
+
+ my $skip = $self->skip_regexes;
File::Find::finddepth(
sub {
- return if ( map { $File::Find::name =~ $_ } @regexen );
+ if( $File::Find::name =~ $skip) {
+ warn "\t- $File::Find::name SKIPPED\n";
+ return;
+ }
+
+
+
&{ $self->on_match() };
} , @{ $self->search_paths||[] }
More information about the Rt-commit
mailing list