[Rt-commit] r5824 - in Mnemonic: . bin ex lib lib/Mnemonic
lib/Mnemonic/Crypto
jesse at bestpractical.com
jesse at bestpractical.com
Tue Aug 29 23:50:02 EDT 2006
Author: jesse
Date: Tue Aug 29 23:50:00 2006
New Revision: 5824
Added:
Mnemonic/bin/
Mnemonic/bin/mnemonic
Mnemonic/ex/
Mnemonic/ex/dotmnemonic_config
Mnemonic/lib/
Mnemonic/lib/Mnemonic/
Mnemonic/lib/Mnemonic.pm
Mnemonic/lib/Mnemonic/Backend/
Mnemonic/lib/Mnemonic/Backend/S3.pm
Mnemonic/lib/Mnemonic/Backend/Tmp.pm
Mnemonic/lib/Mnemonic/Crypto/
Mnemonic/lib/Mnemonic/Crypto/OpenPGP.pm
Mnemonic/lib/Mnemonic/FileSet.pm
Mnemonic/lib/[
Modified:
Mnemonic/ (props changed)
Log:
r23085 at pinglin: jesse | 2006-08-29 23:49:48 -0400
* First Post!
Added: Mnemonic/bin/mnemonic
==============================================================================
--- (empty file)
+++ Mnemonic/bin/mnemonic Tue Aug 29 23:50:00 2006
@@ -0,0 +1,49 @@
+#!/usr/bin/perl -w
+use warnings;
+use strict;
+use Mnemonic;
+use Getopt::Long;
+
+my $b = Mnemonic->new();
+
+my %argv;
+
+
+GetOptions(\%argv,
+ 'restore=s',
+ 'list|ls=s',
+ 'backup',
+ 'index',
+ 'path=s@',
+ 'skip=s@',
+ 'delete_key|delete-key=s',
+ 'dry_run|dry-run'
+ );
+
+
+
+
+$b->init();
+
+ if ($argv{index}) {
+ my @entries = $b->list_backups(prefix => 'MANIFEST');
+ foreach my $entry (@entries) {
+ print $entry."\n";
+ }
+ }
+ if ($argv{delete_key}) {
+ $b->remove_key_from_store($argv{'delete_key'});
+ }
+ if ($argv{backup}) {
+ my @items = $b->list_backups();
+ my $manifest = $b->upload(
+ %argv,
+ stored_keys => \@items
+ );
+ }
+ if ($argv{'list'}) {
+ $b->list_backup_files( manifest_id => $argv{'list'} );
+ }
+ if ($argv{'restore'}) {
+ $b->restore( manifest_id => $argv{'restore'} );
+ }
Added: Mnemonic/ex/dotmnemonic_config
==============================================================================
--- (empty file)
+++ Mnemonic/ex/dotmnemonic_config Tue Aug 29 23:50:00 2006
@@ -0,0 +1,51 @@
+backend: Mnemonic::Backend::Tmp
+S3:
+ access_key: 1FX1111111111
+ secret_key: amazon_secret_access_key_goes_here
+crypto:
+ passphrase: this is a secure passphrase
+ recipients: jesse at fsck.com
+ public_key: "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\
+ Version: GnuPG v1.4.3 (Darwin)\n\n\
+ mQGiBEB4SkkRBACB4GmqpIuJyMles8ApbSfNNobmjx8gDTWmI131NQKoNTugj/DK\n\
+ iKkUHfBYe3PIfsY0X5EKjWy3C/EPze8PH1yu7FMsKrjizHCDgV0748Lz0dwm1gP2\n\
+ p235HIxs8pv2gI1l70GvM0Cqo/xlJ+LqpW/ToJdodHzH2W+ovGbcqTgefwCgzFQg\n\
+ qXVmqaS0myOzBrWPl3KwubkD/0KI5+DkPoTQFxKsMzf1zzT8bJpfAFgvZIgAwaNw\n\
+ GUj+KlmDUauKEeX6FFnRlidpfWKIwYCs23yk5Ypy3jbkHQZp/S1pCvRlAe0FKnry\n\
+ avFBWMonoFxr0edomuM77ljTGLnkOK5L/0G3VPUwTnoMft9WV3c+KT0HQfaDrf/6\n\
+ W0bhA/93XMGQ8lfNFu61dx8+2MCZWSenDRaMv9LupRcXAebFFtIwfN9jKCevQ6Hg\n\
+ 49+UtzehRflEVLrmmZIDd/sUzhUsQN7zGBoYCOgmjsD2tirn4YtErv+TSR01O41v\n\
+ U3R3TRDCFmMmGAoQL5inFxsYpLtB8FNse7bzb1O7wnMZac/lN7QeSmVzc2UgVmlu\n\
+ Y2VudCA8amVzc2VAY3Bhbi5vcmc+iF8EExECAB8FAkB4SrYCGwMHCwkIBwMCAQMV\n\
+ AgMDFgIBAh4BAheAAAoJEBIvXfcQjkBGvssAn2JMeSNholE+JXvVFLgCf6TwuSba\n\
+ AJ9maQCHaMxMj+5zx0RucJQmVrOTirQnSmVzc2UgVmluY2VudCA8amVzc2VAYmVz\n\
+ dHByYWN0aWNhbC5jb20+iGIEExECACICGwMHCwkIBwMCAQMVAgMDFgIBAh4BAheA\n\
+ BQJAeFYGAhkBAAoJEBIvXfcQjkBG/9UAnRly+Z0CP8R5ml4DQYMkfMoDXPK9AKCL\n\
+ nJg9JJnFfi4XH1qxEjKyPvX6qbQeSmVzc2UgVmluY2VudCA8amVzc2VAZnNjay5j\n\
+ b20+iF8EExECAB8FAkB4Sp8CGwMHCwkIBwMCAQMVAgMDFgIBAh4BAheAAAoJEBIv\n\
+ XfcQjkBGCyMAn0d2TFdGNxXNpenytwmVjYXy8bD5AJ0bMr2G9/6c+a3ZT6BpjhZ8\n\
+ E1YMF7kCDQRAeEpUEAgAgFPdBPZUXHyBYPFfa2i3sSHZoenIFwisk72MkWPQX6n8\n\
+ NbiXdS8bQgD/BKlPDep5M47nb6h9rVHYga+d67HJX3Faw/JIdNZNUiNyYKOrICc/\n\
+ BCcJYzRHN4wxkDcX3jRmZnlUz7AjxV6zJ0AKzAoWldFt6/LoGdZJ0Q6SRYWWRjGj\n\
+ 6ei5XtI07+L/ZOh+LtkTpR0r2yjGIwzgzM05wLbgfjI63eDWkii75XOmw74a/MWy\n\
+ e6XegUnljj+6jwoZHDBt6VjHkfpfGTyVl/Lt0bzZpsmvSscg7vQDsG1GDEtrLiBB\n\
+ m4XkOs6ujmYvNYy++DMKJA3EXq6WARzu03fkzFfo9wADBQf9HmPKakGM6ofuvv/r\n\
+ ggxoTomDguh9bFgsn43JJPgTF1QNyWVqE8rFkAXsoUDeNRcJbH4+AADjbO+SCW33\n\
+ f8C4buuyLFjKMwpPh5oMEa9oWXQj+T7JhEjmP3+DYm2p6V7xKYp24jHuXoTegZ5r\n\
+ Z4OAdGjK0O1CGd6lOeOsCXWoX/OYCzoHXu4uVp5BtxZcHLIWbEuP1w0BRLqFZdMm\n\
+ WcvNe+IrE5kSxf50JhQE4JftSSBMKjxErErcPlF0EZCRga4X2erFwshHhrp32nor\n\
+ e5DkHdoIgrKF6Vfq43iJqCCdcIccUYvAf1/fui+3ubfS4mPBgWri7Dvx6YfnXbpq\n\
+ ygbaG4hJBBgRAgAJBQJAeEpUAhsMAAoJEBIvXfcQjkBGeFkAn1FEniiT1IhuXgxw\n\
+ +j7TvcHJaJO5AJ9etgCvbzeRkVWyimqVj3RVxkIfYw==\n\
+ =dsvM\n\
+ -----END PGP PUBLIC KEY BLOCK-----\n"
+
+set:
+ name: home
+ include: $ENV{'HOME'}
+ exclude:
+ - '*.bak'
+ - '*~'
+ - '#*'
+ - 'tmp/'
+ - 'var/'
Added: Mnemonic/lib/Mnemonic.pm
==============================================================================
--- (empty file)
+++ Mnemonic/lib/Mnemonic.pm Tue Aug 29 23:50:00 2006
@@ -0,0 +1,239 @@
+#!/usr/bin/perl -w
+use warnings;
+use strict;
+
+
+package Mnemonic;
+
+use IO::All;
+use Digest::SHA qw(sha256);
+use Digest::MD5 qw(md5_hex);
+use YAML::Syck;
+use File::Path;
+use UNIVERSAL::require;
+use Mnemonic::FileSet;
+use Mnemonic::Crypto::OpenPGP;
+
+use base qw/Class::Accessor/;
+
+BEGIN {__PACKAGE__->mk_accessors(qw/backend config_file config pgp/);}
+
+
+sub init {
+ my $self = shift;
+ $self->load_config();
+
+ $self->pgp( Mnemonic::Crypto::OpenPGP->new({config => $self->config}));
+ my $backend = $self->config->{'backend'} || "Mnemonic::Backend::Tmp";
+ $backend->require() || die $@;
+ $self->backend( $backend->new({ config => $self->config()}) );
+ $self->backend->init();
+}
+
+sub load_config {
+ my $self = shift;
+ for ($self->config_file, $ENV{'MNEMONIC_CONFIG'}, $ENV{'HOME'}."/.mnemonic_config", "/etc/mnemonic_config") {
+ if ( $_ and -f $_) {
+ eval {
+ $self->config(YAML::Syck::LoadFile($_));
+ }; warn $@ if( $@);
+ }
+ }
+ unless ($self->config){
+ $self->config({});
+ }
+
+}
+
+sub list_backups {
+ my $self = shift;
+ return ( $self->backend->get_keys(@_) );
+}
+
+sub upload {
+ my $self = shift;
+ my %args = (
+ stored_keys => undef,
+ path => undef,
+ skip => undef,
+ dry_run => undef,
+ @_
+ );
+
+ my %manifest;
+
+ my $search = Mnemonic::FileSet->new({ search_paths => $args{'path'}, skip_patterns => $args{'skip'}});
+ my @manifest = $search->search();
+ # store a key with a content-type and some optional metadata
+
+
+ my $seen_sha256 = {};
+ $seen_sha256->{$_} = "prestored" for (@{$args{'stored_keys'}});
+
+ foreach my $filename ( @manifest ) {
+ warn "\t$filename\n";
+ next if ($args{dry_run});
+ my $sha256= $self->store_file(manifest =>\%manifest, filename => $filename, stored_keys => $seen_sha256);
+ $seen_sha256->{$sha256} = $filename
+ }
+ return undef if ($args{'dry_run'});
+
+ my $hostname = `hostname`;
+ chomp $hostname;
+ my $manifest_id = 'MANIFEST/' . $hostname . "/" . time() . '-' . $$;
+ $self->backend->store( $manifest_id => YAML::Syck::Dump( \%manifest ) );
+ return $manifest_id;
+}
+
+ sub store_file {
+ my $self = shift;
+ my %args = ( manifest => undef,
+ filename => undef,
+ @_);
+
+ my $filename = $args{'filename'};
+ my $manifest = $args{'manifest'};
+
+ my @path = File::Spec->splitdir($filename);
+ pop @path;
+ shift @path;
+ my $path = '';
+ while ( my $dir = shift @path ) {
+ $path = File::Spec->catdir( $path, $dir );
+ next unless -d $path;
+ next if ( exists $manifest->{$path} );
+
+ $manifest->{$path} = {
+ type => 'directory',
+ statinfo => [ stat($path) ],
+ stored => '0'
+ }
+
+ }
+ if (-d $filename) {
+ $manifest->{$filename} = {
+ type => 'directory',
+ ttatinfo => [ stat($filename) ],
+ stored => '0'
+ }
+
+ } elsif (-f $filename) {
+ if (-z $filename) {
+ $manifest->{$filename} = {
+ type => 'file',
+ size => '0',
+ statinfo => [ stat($filename) ],
+ stored => '0'
+ }
+
+
+ } else {
+ eval {
+ my $sha256_sum = _get_sha256($filename);
+ if ( $args{'stored_keys'}->{ $sha256_sum } ) {
+ warn "\t found $sha256_sum on the server. passing on it\n";
+
+ } else {
+ warn "\t as $sha256_sum\n";
+ my $plaintext < io $filename;
+ my $cyphertext = $self->pgp->encrypt( $plaintext );
+ $self->backend->store( $sha256_sum => $cyphertext, );
+ }
+
+
+ my @stat = stat($filename);
+
+
+ $manifest->{$filename} = {
+ key => $sha256_sum,
+ statinfo => \@stat,
+ stored => 1
+ };
+ }; if ($@){
+ $manifest->{'!errors'}->{$filename} = $@;
+ }
+ }
+ }
+}
+
+sub restore {
+ my $self = shift;
+ my %args = (
+ manifest_id => undef,
+ restore_root => undef,
+ @_
+ );
+
+ $args{'restore_root'} = File::Spec->catdir( '/tmp', $args{'manifest_id'} );
+
+ my $root = $args{'restore_root'};
+ File::Path::mkpath( [$root] );
+
+ my $result = $self->backend->fetch( $args{'manifest_id'} );
+ #{ content_type, etag, value, @meta }
+ my $manifest = YAML::Syck::Load( $result->{value} );
+
+ foreach my $file ( sort keys %$manifest ) {
+ my @path = File::Spec->splitdir($file);
+ pop @path;
+ my $dir = File::Spec->catdir( $root, @path );
+ unless ( -d $dir ) {
+ File::Path::mkpath( [$dir] );
+
+ }
+ my $target = File::Spec->catfile( $root, $file );
+ warn "Fetching $file\n";
+ if (($manifest->{$file}->{stored}||0) > 0 ) {
+ warn "...it's a file";
+ my $file_result = $self->backend->fetch( $manifest->{$file}->{key} );
+ my $pt = $self->pgp->decrypt( $file_result->{'value'} => $target);
+ unless ( $manifest->{$file}->{key} eq _get_sha256($target) ) {
+ warn "$file has an invalid SHA256 sum after decryption";
+ }
+ } else {
+ warn "Not fetching $file\n";
+
+ }
+ if ((-f $target or -d $target) && $manifest->{$file}->{statinfo} ) {
+ my @stat = @{ $manifest->{$file}->{statinfo} };
+ my $mode = $stat[2] & 07777;
+ chmod $mode, $target || die $@;
+ utime $stat[8], $stat[9], $target;
+ chown $stat[4], $stat[5], $target;
+ print "Wrote $file to $target\n";
+ }
+ }
+}
+
+sub remove_key_from_store {
+ my $self = shift;
+ my %args = (key => undef,
+ @_);
+
+ $self->backend->delete($args{'key'});
+
+}
+
+
+sub list_backup_files {
+ my $self = shift;
+ my %args = (@_);
+
+ my $result = $self->backend->fetch( $args{'manifest_id'} );
+ my $manifest = YAML::Syck::Load( $result->{value} );
+ warn "This backup would restore:\n";
+ foreach my $file (sort keys %$manifest) {
+ print $file ."\n";
+ }
+
+}
+
+sub _get_sha256 {
+ my $filename = shift;
+ my $sha = Digest::SHA->new('sha256');
+
+ $sha->addfile($filename); # feed data into stream
+ return $sha->hexdigest;
+}
+
+1;
Added: Mnemonic/lib/Mnemonic/Backend/S3.pm
==============================================================================
--- (empty file)
+++ Mnemonic/lib/Mnemonic/Backend/S3.pm Tue Aug 29 23:50:00 2006
@@ -0,0 +1,83 @@
+use warnings;
+use strict;
+
+package Mnemonic::Backend::S3;
+use Net::Amazon::S3;
+
+use base 'Class::Accessor';
+
+BEGIN{__PACKAGE__->mk_accessors(qw/s3 bucket config/)};
+
+use vars qw/$OWNER_ID $OWNER_DISPLAYNAME/;
+
+
+sub init {
+ my $self = shift;
+ my $aws_access_key_id = $self->config->{'S3'}{'access_key'}
+ || die "S3 access_key is not set in your config";
+
+ my $aws_secret_access_key = $self->config->{'S3'}{'secret_key'}
+ || die "S3 secret_key is not set in your config";
+ $self->s3(
+ Net::Amazon::S3->new(
+ { aws_access_key_id => $aws_access_key_id,
+ aws_secret_access_key => $aws_secret_access_key,
+ }
+ )
+ );
+
+ # you can also pass a timeout in seconds
+
+ my $hostname = `hostname`;
+ chomp($hostname);
+ my $bucketname = 'bps_s3_client-0.3-' . $hostname;
+
+ my $bucket_list = $self->s3->buckets();
+
+ $self->bucket(
+ ( grep { $_->{bucket} eq $bucketname }
+ @{ $bucket_list->{'buckets'} }
+ )[0]
+ );
+ unless ( $self->bucket ) {
+
+ $self->bucket( $self->s3->add_bucket( { bucket => $bucketname } ) )
+ or die $self->s3->err . ": " . $self->s3->errstr;
+ }
+
+}
+
+sub get_keys {
+ my $self = shift;
+ my @keys;
+ my $fetch_more = 1;
+ my $marker = '';
+ while ($fetch_more) {
+ my $response = $self->bucket->list( {'max-keys' => 1000, marker => $marker, @_});
+ push @keys , @{ $response->{keys} ||[]};
+ last unless ($response->{'is_truncated'});
+ $marker = $keys[-1]->{'key'};
+ }
+ return (map {$_->{key}} @keys);
+}
+
+sub store {
+ my $self = shift;
+ $self->bucket->add_key( @_);
+}
+
+
+sub delete {
+ my $self = shift;
+ my $key = shift;
+ return $self->bucket->get_key($key);
+
+}
+sub fetch {
+ my $self = shift;
+ return $self->bucket->get_key(@_);
+
+}
+
+
+1;
Added: Mnemonic/lib/Mnemonic/Backend/Tmp.pm
==============================================================================
--- (empty file)
+++ Mnemonic/lib/Mnemonic/Backend/Tmp.pm Tue Aug 29 23:50:00 2006
@@ -0,0 +1,63 @@
+#!/usr/bin/perl -w
+use warnings;
+use strict;
+
+package Mnemonic::Backend::Tmp;
+use IO::All;
+use File::Spec;
+use Digest::MD5;
+use base 'Class::Accessor';
+use File::Find::Rule;
+
+__PACKAGE__->mk_accessors(qw/path/);
+
+sub init {
+ my $self = shift;
+ $self->path('/tmp/blinus');
+ mkdir ($self->path);
+}
+
+sub get_keys {
+ my $self = shift;
+ my %args = (prefix => '',
+ @_);
+ my @files = File::Find::Rule->file()
+ ->in($self->path );
+ my $path = $self->path;
+ my $matching = $args{prefix};
+ return (grep { $_ =~ s/^$path\/// && $_ =~ /^$matching/ } @files);
+}
+
+sub store {
+ my $self = shift;
+ my $key = shift;
+ my $path = $self->resolve_key($key);
+ my @path = File::Spec->splitdir($path);
+ pop(@path);
+ my $indir = File::Spec->catdir('/', at path);
+ File::Path::mkpath( [$indir
+ ] );
+
+ open (my $outfile, ">", $path) || die "Can't open $path ". $!;
+ print $outfile $_[0] || die $!;
+ close $outfile || die $!;
+}
+
+sub fetch {
+ my $self = shift;
+ my $key = shift;
+ my $tmp;
+ io($self->resolve_key($key)) > $tmp;
+ return { value => $tmp,
+ etag => Digest::MD5::md5_hex($tmp)
+ };
+
+}
+
+sub resolve_key {
+ my $self = shift;
+ my $key = shift;
+ return File::Spec->catpath('',$self->path,$key);
+}
+
+1;
Added: Mnemonic/lib/Mnemonic/Crypto/OpenPGP.pm
==============================================================================
--- (empty file)
+++ Mnemonic/lib/Mnemonic/Crypto/OpenPGP.pm Tue Aug 29 23:50:00 2006
@@ -0,0 +1,69 @@
+#!/usr/bin/perl -w
+use warnings;
+use strict;
+
+
+package Mnemonic::Crypto::OpenPGP;
+
+ use Crypt::OpenPGP;
+ use Crypt::OpenPGP::KeyRing;
+
+use base qw/Class::Accessor/;
+
+BEGIN {__PACKAGE__->mk_accessors(qw/config pgp public_keyring/);}
+
+
+sub new {
+ my $class = shift;
+ my $self = $class->SUPER::new(@_);
+ $self->public_keyring(
+ Crypt::OpenPGP::KeyRing->new(
+ Data => $self->config->{'crypto'}->{'public_key'}
+ )
+ );
+ my $pgp = Crypt::OpenPGP->new(
+ Compat => 'GnuPG',
+ PubRing => $self->public_keyring
+ );
+
+ $self->pgp($pgp);
+ return $self;
+}
+
+
+sub encrypt {
+ my $self = shift;
+ my $plaintext = shift;
+ warn "here";
+ my $cyphertext;
+ eval {
+ $cyphertext = $self->pgp->encrypt(
+ Data => $plaintext,
+ Recipients => $self->config->{'crypto'}->{'recipients'},
+ Compress => 'ZIP'
+ ) || die $self->pgp->errstr;
+ };
+ return $cyphertext;
+}
+
+
+
+sub decrypt {
+ my $self = shift;
+ my $cyphertext = shift;
+ my $target = shift;
+ my $pt = $self->pgp->decrypt(
+ Data => $cyphertext,
+ Passphrase => $self->config->{'crypto'}->{'passphrase'}
+ ) || die $self->pgp->errstr;
+ open( my $outfile, ">", $target ) || die $!;
+ binmode($outfile);
+ print $outfile $pt || die $!;
+ close $outfile || die $!;
+}
+
+
+
+
+
+1;
Added: Mnemonic/lib/Mnemonic/FileSet.pm
==============================================================================
--- (empty file)
+++ Mnemonic/lib/Mnemonic/FileSet.pm Tue Aug 29 23:50:00 2006
@@ -0,0 +1,69 @@
+use warnings;
+use strict;
+
+package Mnemonic::FileSet;
+
+use base qw/Class::Accessor/;
+
+__PACKAGE__->mk_accessors(qw(skip_patterns skip_regexes search_paths ));
+
+use File::Find;
+
+
+
+sub init {
+ my $self = shift;
+ my @regexen;
+ foreach my $entry (@{$self->skip_patterns||[]}) {
+ $entry =~ s/(\.|\(|\))/\\$1/g;
+ $entry =~ s/\*/\(\?\:\.\*\)/g;
+ $entry = "\/$entry";
+ warn $entry;
+ push @regexen, qr/$entry/i;
+ }
+ $self->skip_regexes(@regexen||[]);
+
+
+}
+
+sub search {
+ my $self = shift;
+ $self->init();
+ my @files;
+ my @regexen = @{$self->skip_regexes};
+ $|++; # turn off buffering
+ my $count = 0;
+ my $skipped = 0;
+ my $last = 0;
+ print "Looking:\n\n";
+ File::Find::finddepth(
+
+sub {
+ my $name = $File::Find::name;
+ if ($count or $skipped) {
+ print "\b" x $last;
+ }
+ if (map { $name =~ $_ } @regexen) {
+ ++$skipped;
+ } else {
+ ++$count;
+ push @files, $name ;
+ }
+ my $line = "Backing up $count files. Skipping $skipped: $name";
+ print $line;
+ if ($last - length($line) > 0) {
+
+ print " " x ($last - length($line));
+ } else {
+ $last = length($line);
+ }
+
+}
+
+
+ , @{$self->search_paths});
+ print "\nDone";
+ return @files;
+}
+
+1;
Added: Mnemonic/lib/[
==============================================================================
More information about the Rt-commit
mailing list