[Rt-commit] r5842 - in Mnemonic: . bin lib lib/Mnemonic
lib/Mnemonic/Crypto
jesse at bestpractical.com
jesse at bestpractical.com
Sun Sep 3 00:50:52 EDT 2006
Author: jesse
Date: Sun Sep 3 00:50:51 2006
New Revision: 5842
Modified:
Mnemonic/ (props changed)
Mnemonic/bin/mnemonic
Mnemonic/lib/Mnemonic.pm
Mnemonic/lib/Mnemonic/Backend/Tmp.pm
Mnemonic/lib/Mnemonic/Crypto/OpenPGP.pm
Mnemonic/lib/Mnemonic/FileSet.pm
Log:
r26989 at pinglin: jesse | 2006-09-03 00:51:55 -0400
* seems to work
Modified: Mnemonic/bin/mnemonic
==============================================================================
--- Mnemonic/bin/mnemonic (original)
+++ Mnemonic/bin/mnemonic Sun Sep 3 00:50:51 2006
@@ -8,7 +8,6 @@
my %argv;
-
GetOptions(\%argv,
'restore=s',
'list|ls=s',
Modified: Mnemonic/lib/Mnemonic.pm
==============================================================================
--- Mnemonic/lib/Mnemonic.pm (original)
+++ Mnemonic/lib/Mnemonic.pm Sun Sep 3 00:50:51 2006
@@ -6,17 +6,19 @@
use IO::All;
use Digest::SHA qw(sha256);
use Digest::MD5 qw(md5_hex);
-use YAML::Syck ();
-use File::Path ();
-use UNIVERSAL::require ();
-use Mnemonic::FileSet ();
+use YAML::Syck ();
+use File::Path ();
+use UNIVERSAL::require ();
+use Mnemonic::FileSet ();
use Mnemonic::Crypto::OpenPGP ();
-use Shell::Command ();
+use Shell::Command ();
use base qw/Class::Accessor/;
BEGIN { __PACKAGE__->mk_accessors(qw/backend config_file config pgp/); }
+our $CHUNKSIZE = 1024000;
+
sub init {
my $self = shift;
$self->load_config();
@@ -62,37 +64,33 @@
@_
);
- my %manifest;
-
+ my $manifest = {};
+ my $seen_checksum = {};
+ $seen_checksum->{$_} = "prestored" for ( @{ $args{'stored_keys'} } );
+ my $on_match= sub {
+ my $checksum = $self->store_item( manifest => $manifest, filename => $File::Find::name, stored_keys => $seen_checksum);
+ $seen_checksum->{$checksum} = $File::Find::name;
+ };
my $search = Mnemonic::FileSet->new(
- { search_paths => $args{'path'}, skip_patterns => $args{'skip'} } );
+ { search_paths => $args{'path'}, skip_patterns => $args{'skip'} ,
+ on_match => $on_match
+ },
+
+ );
my @manifest = $search->search();
- # store a key with a content-type and some optional metadata
-
- my $seen_sha256 = {};
- $seen_sha256->{$_} = "prestored" for ( @{ $args{'stored_keys'} } );
-
- foreach my $filename (@manifest) {
- next if ( $args{dry_run} );
- my $sha256 = $self->store_file(
- manifest => \%manifest,
- filename => $filename,
- stored_keys => $seen_sha256
- );
- $seen_sha256->{$sha256} = $filename;
- }
- return undef if ( $args{'dry_run'} );
my $hostname = `hostname`;
chomp $hostname;
- my $manifest_id = 'MANIFEST/' . $hostname . "/" . time() . '-' . $$;
- $self->backend->store( $manifest_id => YAML::Syck::Dump( \%manifest ) );
+ my $manifest_id = $self->pgp->key_id .'/MANIFEST/' . $hostname . "/" . time() . '-' . $$;
+ my $manifest_yaml = YAML::Syck::Dump($manifest);
+
+ $self->backend->store( $manifest_id => $self->pgp->encrypt($manifest_yaml) );
warn "Completed backup $manifest_id\n";
return $manifest_id;
}
-sub store_file {
+sub store_item {
my $self = shift;
my %args = (
manifest => undef,
@@ -104,9 +102,11 @@
my $manifest = $args{'manifest'};
my @path = File::Spec->splitdir($filename);
- pop @path;
- shift @path;
+ pop @path; shift @path; # remove the filename and the leading slash
my $path = '';
+
+
+
while ( my $dir = shift @path ) {
$path = File::Spec->catdir( $path, $dir );
next unless -d $path;
@@ -119,10 +119,12 @@
}
}
+
+
if ( -d $filename ) {
$manifest->{$filename} = {
type => 'directory',
- ttatinfo => [ stat($filename) ],
+ statinfo => [ stat($filename) ],
stored => '0'
}
@@ -131,39 +133,49 @@
$manifest->{$filename} = {
type => 'file',
size => '0',
- statinfo => [ stat($filename) ],
+ statinfo => [ stat $filename ],
stored => '0'
}
} else {
+
+
eval {
- my $sha256_sum = _get_sha256($filename);
- if ( $args{'stored_keys'}->{$sha256_sum} ) {
- warn "\t= $filename already stored\n";
-
- } else {
- my $plaintext < io $filename;
- my $cyphertext = $self->pgp->encrypt($plaintext);
- eval {
- $self->backend->store( $sha256_sum => $cyphertext, );
- warn "\t+ $filename uploaded\n";
- };
- if ($@) {
- warn "\t!ERROR: $filename: $@\n";
- }
- }
- my @stat = stat($filename);
+ my @statinfo = stat $filename;
+
+ my $len = $statinfo[7];
+ my $start_at = 0;
+ my @subkeys;
+ open(my $fh, "<", $filename) || die $!;
+ while ( $start_at <= $len ) {
+ my $chunk_ref = $self->read_chunk( $fh, $CHUNKSIZE );
+ my $checksum = $self->_get_checksum($chunk_ref);
+ if ( $args{'stored_keys'}->{$checksum} ) {
+ warn "\t= $filename already stored\n";
+ } else {
+ $self->encrypt_and_store($checksum => $chunk_ref);
+ warn "\t+ $filename (from " . $start_at . ") uploaded\n";
+ }
+ push @subkeys, $checksum;
+ $start_at += $CHUNKSIZE;
+ }
+ close ($fh);
+
+
+
$manifest->{$filename} = {
- key => $sha256_sum,
- statinfo => \@stat,
+ keys => \@subkeys,
+ statinfo => \@statinfo,
stored => 1,
- type => 'file'
+ sha256 => $self->_sha256($filename),
+ type => 'file'
};
};
if ($@) {
$manifest->{'!errors'}->{$filename} = $@;
+ warn $@;
}
}
}
@@ -177,14 +189,15 @@
@_
);
- $args{'restore_root'} ||= File::Spec->catdir( '/tmp', $args{'manifest_id'} );
+ $args{'restore_root'}
+ ||= File::Spec->catdir( '/tmp', $args{'manifest_id'} );
my $root = $args{'restore_root'};
File::Path::mkpath( [$root] );
my $result = $self->backend->fetch( $args{'manifest_id'} );
#{ content_type, etag, value, @meta }
- my $manifest = YAML::Syck::Load( $result->{value} );
+ my $manifest = YAML::Syck::Load( $self->pgp->decrypt( $result->{'value'}));
foreach my $file ( sort keys %$manifest ) {
my @path = File::Spec->splitdir($file);
@@ -197,47 +210,57 @@
my $target = File::Spec->catfile( $root, $file );
- if ($manifest->{$file}->{type} eq 'file') {
- $self->restore_file($file => $target => $manifest->{$file});
- } elsif($manifest->{$file}->{type} eq 'directory') {
- $self->restore_directory($file => $target);
+ if ( $manifest->{$file}->{type} eq 'file' ) {
+ $self->restore_file( $file => $target => $manifest->{$file} );
+ } elsif ( $manifest->{$file}->{type} eq 'directory' ) {
+ $self->restore_directory( $file => $target );
} else {
- warn "WTF $file". YAML::Syck::Dump($manifest->{$file});;
+ warn "WTF $file" . YAML::Syck::Dump( $manifest->{$file} );
}
$self->update_statinfo( $target => $manifest->{$file}->{statinfo} );
warn "\t+ Wrote $target\n";
}
}
-
sub restore_directory {
- my $self =shift;
- my $dir = shift;
- my $target = shift;
+ my $self = shift;
+ my $dir = shift;
+ my $target = shift;
- File::Path::mkpath( [$target] );
+ File::Path::mkpath( [$target] );
}
sub restore_file {
- my $self =shift;
- my $file = shift;
- my $target = shift;
- my $manifest_entry = shift;
-
- if ( ( $manifest_entry->{stored} || 0 ) > 0 ) {
- my $file_result = $self->backend->fetch( $manifest_entry->{key} );
- my $pt = $self->pgp->decrypt( $file_result->{'value'} => $target );
- unless ( $manifest_entry->{key} eq _get_sha256($target) ) {
- warn "$file has an invalid SHA256 sum after decryption";
- }
- } elsif ($manifest_entry->{size} ==0 ) {
- Shell::Command::touch $target;
+ my $self = shift;
+ my $file = shift;
+ my $target = shift;
+ my $manifest_entry = shift;
+
+ if ( ( $manifest_entry->{stored} || 0 ) > 0 ) {
+
+ open( my $outfile, ">", $target ) || die $!;
+ binmode($outfile);
+ foreach my $key (@{$manifest_entry->{keys}}) {
+
+ my $file_result = $self->backend->fetch( $key) ;
+ my $pt = $self->pgp->decrypt( $file_result->{'value'});
+ warn "\t\trestoring ".$key."\n";
+ print $outfile $pt || die $!;
+ }
- } else{
- warn "Not fetching $file\n";
+ close $outfile || die $!;
+ unless ( $manifest_entry->{sha256} eq $self->_sha256($target) ) {
+ warn "$file has an invalid SHA256 sum after decryption";
}
+ } elsif ( $manifest_entry->{size} == 0 ) {
+ Shell::Command::touch $target;
+
+ } else {
+ warn "Not fetching $file\n";
+
+ }
}
sub remove_key_from_store {
@@ -264,35 +287,79 @@
}
-sub _get_sha256 {
- my $filename = shift;
+sub _sha256 {
+ my $self =shift;
+ my $item = shift;
my $sha = Digest::SHA->new('sha256');
- $sha->addfile($filename); # feed data into stream
- return $sha->hexdigest;
-}
-
-
-
+ if (ref($item) eq 'SCALAR') {
+ $sha->add($$item);
+ } else {
+ $sha->addfile($item); # feed data into stream
+ }
+ return $sha->hexdigest;
- sub update_statinfo {
- my $self = shift;
- my $target = shift;
- my $info = shift;
+}
+sub _get_checksum {
+ my $self = shift;
+ return $self->pgp->key_id ."/". $self->_sha256(@_);
+}
+sub update_statinfo {
+ my $self = shift;
+ my $target = shift;
+ my $info = shift;
-if ( ( -f $target or -d $target ) && $info) {
+ if ( ( -f $target or -d $target ) && $info ) {
{
- my @stat = @{ $info};
+ my @stat = @{$info};
my $mode = $stat[2] & 07777;
chmod $mode, $target || die $@;
- utime $stat[8], $stat[9], $target;
chown $stat[4], $stat[5], $target;
+ utime $stat[8], $stat[9], $target;
}
}
+}
+
+
+
+
+
+=head2 read_chunk filename offset length
+
+returns a reference to the content it got.
+
+=cut
+
+
+sub read_chunk {
+ my $self = shift;
+ my $fh =shift;
+ my $length = shift;
+
+ my $content;
+ binmode($fh)||die $!;
+ read($fh, $content, $length)||die $!;
+ return \$content;
+}
+
+
+
+ sub encrypt_and_store {
+ my $self = shift;
+ my $key = shift;
+ my $plaintext_ref = shift;
+
+ my $cyphertext = $self->pgp->encrypt($$plaintext_ref);
+ eval {
+ $self->backend->store( $key => $cyphertext, );
+ };
+ if ($@) {
+ warn "\t!ERROR: $key: $@\n";
+ }
+ }
- }
1;
Modified: Mnemonic/lib/Mnemonic/Backend/Tmp.pm
==============================================================================
--- Mnemonic/lib/Mnemonic/Backend/Tmp.pm (original)
+++ Mnemonic/lib/Mnemonic/Backend/Tmp.pm Sun Sep 3 00:50:51 2006
@@ -13,7 +13,7 @@
sub init {
my $self = shift;
- $self->path('/tmp/blinus');
+ $self->path('/tmp/mnemonic');
mkdir ($self->path);
}
Modified: Mnemonic/lib/Mnemonic/Crypto/OpenPGP.pm
==============================================================================
--- Mnemonic/lib/Mnemonic/Crypto/OpenPGP.pm (original)
+++ Mnemonic/lib/Mnemonic/Crypto/OpenPGP.pm Sun Sep 3 00:50:51 2006
@@ -50,19 +50,21 @@
sub decrypt {
my $self = shift;
my $cyphertext = shift;
- my $target = shift;
my $pt = $self->pgp->decrypt(
Data => $cyphertext,
Passphrase => $self->config->{'crypto'}->{'passphrase'}
) || die $self->pgp->errstr;
- open( my $outfile, ">", $target ) || die $!;
- binmode($outfile);
- print $outfile $pt || die $!;
- close $outfile || die $!;
+
+ return $pt;
}
+sub key_id {
+ my $self = shift;
+ return unpack('H*',$self->public_keyring->find_keyblock_by_uid($self->config->{'crypto'}->{'recipients'})->key->key_id);
+
+}
1;
Modified: Mnemonic/lib/Mnemonic/FileSet.pm
==============================================================================
--- Mnemonic/lib/Mnemonic/FileSet.pm (original)
+++ Mnemonic/lib/Mnemonic/FileSet.pm Sun Sep 3 00:50:51 2006
@@ -5,7 +5,7 @@
use base qw/Class::Accessor/;
-__PACKAGE__->mk_accessors(qw(skip_patterns skip_regexes search_paths ));
+__PACKAGE__->mk_accessors(qw(skip_patterns skip_regexes search_paths on_match));
use File::Find;
@@ -36,27 +36,9 @@
my $skipped = 0;
my $last = 0;
print "Looking:\n\n";
- File::Find::finddepth(
-
-sub {
- my $name = $File::Find::name;
- if ($count or $skipped) {
- print "\b" x $last;
- }
- if (map { $name =~ $_ } @regexen) {
- ++$skipped;
- } else {
- ++$count;
- push @files, $name ;
- }
- my $line = "Backing up $count files. Skipping $skipped: $name";
- print $line;
- if ($last - length($line) > 0) {
-
- print " " x ($last - length($line));
- } else {
- $last = length($line);
- }
+ File::Find::finddepth( sub {
+ return if (map { $File::Find::name =~ $_ } @regexen);
+ &{$self->on_match()};
}
More information about the Rt-commit
mailing list