[Bps-public-commit] Prophet branch, master, updated. b39f00a9f3e3f9bf6f951713816a9c5393b4a07d
jesse
jesse at bestpractical.com
Thu Apr 16 05:28:39 EDT 2009
The branch, master has been updated
via b39f00a9f3e3f9bf6f951713816a9c5393b4a07d (commit)
via 3397d8bfb085699a272a2d59ca53f20f3ef8b632 (commit)
via 6a0f13e9979ad24725ab83ac60a7097e5922c12d (commit)
via 474629303b770961e07dd67da1e10cef3fad98df (commit)
via 3700b698f0c3879aa2dc0dc57650a3b9e70c704f (commit)
via 4ac865b973c081a92b2cf627508991fdbee6dd6f (commit)
via 1d0966f8123f2de6bf3336071126257a0a876bf7 (commit)
via aaa72b7b269123ae23bb40b202e826f87b86b74e (commit)
via 35db2ef4ee46a1511b840e02d8d4ceae422d1c5a (commit)
via 6707102cd5efd50b841a1cba1766af6f4399f769 (commit)
via 9ee21e860d11f7e95fbb16548f1f005f6c11eabb (commit)
via 3df6c485de55632e31c43389c0e4f8724f08c74e (commit)
from 2b39c79c80b84fc21d088ae0eec698f8c8481b44 (commit)
Summary of changes:
lib/Prophet/CLI/Command/Clone.pm | 6 +---
lib/Prophet/CLI/Command/Merge.pm | 14 +++++----
lib/Prophet/CLI/MirrorCommand.pm | 4 +-
lib/Prophet/CLI/ProgressBar.pm | 3 +-
lib/Prophet/FilesystemReplica.pm | 3 --
lib/Prophet/Replica.pm | 2 +-
lib/Prophet/Replica/prophet_cache.pm | 51 ++++++++++++-------------------
lib/Prophet/Replica/sqlite.pm | 55 ++++++++++++++++++++++++++++-----
8 files changed, 79 insertions(+), 59 deletions(-)
- Log -----------------------------------------------------------------
commit 3df6c485de55632e31c43389c0e4f8724f08c74e
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 14:07:59 2009 +0800
Record changeset sha1 when using the sqlite replica type
(This will make it possible to generate a changesets.idx)
diff --git a/lib/Prophet/Replica/sqlite.pm b/lib/Prophet/Replica/sqlite.pm
index cd2edf6..f6115ae 100644
--- a/lib/Prophet/Replica/sqlite.pm
+++ b/lib/Prophet/Replica/sqlite.pm
@@ -6,6 +6,8 @@ use File::Spec ();
use Data::UUID;
use File::Path;
use Prophet::Util;
+use JSON;
+use Digest::SHA1 qw/sha1_hex/;
use DBI;
has dbh => (
@@ -96,9 +98,8 @@ sub BUILD {
sub _check_for_upgrades {
my $self = shift;
- if ( $self->replica_version && $self->replica_version < 2) {
- $self->_upgrade_replica_to_v2();
- }
+ if ( $self->replica_version && $self->replica_version < 2) { $self->_upgrade_replica_to_v2(); }
+ if ( $self->replica_version && $self->replica_version < 3) { $self->_upgrade_replica_to_v3(); }
}
@@ -253,7 +254,8 @@ CREATE TABLE changesets (
is_resolution boolean,
original_source_uuid text,
- original_sequence_no INTEGER
+ original_sequence_no INTEGER,
+ sha1 TEXT
)
}, q{
CREATE TABLE changes (
@@ -294,7 +296,7 @@ CREATE TABLE userdata (
$self->set_db_uuid( $args{'db_uuid'} || Data::UUID->new->create_str );
$self->set_replica_uuid( Data::UUID->new->create_str );
- $self->set_replica_version(2);
+ $self->set_replica_version(3);
$self->resolution_db_handle->initialize( db_uuid => $args{resdb_uuid} )
if !$self->is_resdb;
$self->after_initialize->($self);
@@ -464,7 +466,7 @@ sub _load_changeset_from_db {
my $sth = $self->dbh->prepare("SELECT creator, created, sequence_no, ".
"original_source_uuid, original_sequence_no, ".
- "is_nullification, is_resolution from changesets ".
+ "is_nullification, is_resolution, sha1 from changesets ".
"WHERE sequence_no = ?");
$sth->execute($args{sequence_no});
@@ -477,6 +479,7 @@ sub _instantiate_changeset_from_db {
my $self = shift;
my $data = shift;
require Prophet::ChangeSet;
+ my $sha1 = delete $data->{sha1};
my $changeset = Prophet::ChangeSet->new(%$data, source_uuid => $self->uuid );
@@ -496,6 +499,11 @@ sub _instantiate_changeset_from_db {
push @{$changeset->changes}, $change;
}
+ if(!$sha1) {
+ my $update_sth = $self->dbh->prepare('UPDATE changesets set sha1 = ? where sequence_no = ?');
+ $update_sth->execute($self->_calculate_changeset_sha1($changeset), $changeset->sequence_no);
+ }
+
return $changeset;
}
@@ -537,21 +545,38 @@ sub commit_edit {
$self->current_edit(undef);
}
+sub _calculate_changeset_sha1 {
+my $self = shift;
+my $changeset = shift;
+ my $hash_changeset = $changeset->as_hash;
+ # These two things should never actually get stored
+ my $seqno = delete $hash_changeset->{'sequence_no'};
+ my $uuid = delete $hash_changeset->{'source_uuid'};
+
+ my $sha1 = sha1_hex(to_json( $hash_changeset,
+ { canonical => 1, pretty => 0, utf8 => 1 } ));
+
+ return $sha1;
+}
+
sub _write_changeset_to_db {
my $self = shift;
my $changeset = shift;
+ my $sha1 = $self->_calculate_changeset_sha1($changeset);
+
$self->dbh->do(
"INSERT INTO changesets "
. "(creator, created,"
. "original_source_uuid, original_sequence_no, "
- . "is_nullification, is_resolution) "
+ . "is_nullification, is_resolution, sha1) "
. "VALUES(?,?,?,?,?,?)", {},
$changeset->creator, $changeset->created,
$changeset->original_source_uuid,
$changeset->original_sequence_no, $changeset->is_nullification,
- $changeset->is_resolution
+ $changeset->is_resolution,
+ $sha1
);
@@ -812,6 +837,18 @@ sub _upgrade_replica_to_v2 {
);
}
+sub _upgrade_replica_to_v3 {
+ my $self = shift;
+
+ $self->_do_db_upgrades(
+ statements => [
+ q{ALTER TABLE changesets ADD COLUMN sha1 text}
+ ],
+ version => 3
+ );
+
+}
+
sub _do_db_upgrades {
my $self = shift;
commit 9ee21e860d11f7e95fbb16548f1f005f6c11eabb
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 14:08:51 2009 +0800
Remove a comment which is now a lie
diff --git a/lib/Prophet/FilesystemReplica.pm b/lib/Prophet/FilesystemReplica.pm
index a211932..c098b0c 100644
--- a/lib/Prophet/FilesystemReplica.pm
+++ b/lib/Prophet/FilesystemReplica.pm
@@ -116,9 +116,6 @@ sub _write_changeset {
my $fh = $args{'index_handle'};
my $hash_changeset = $changeset->as_hash;
-
-# XXX TODO: we should not be calculating the changeset's sha1 with the 'source_uuid' and 'sequence_no' inside it. that makes every replica have a different hash for what should be the samechangeset.
-
# These two things should never actually get stored
my $seqno = delete $hash_changeset->{'sequence_no'};
my $uuid = delete $hash_changeset->{'source_uuid'};
commit 6707102cd5efd50b841a1cba1766af6f4399f769
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 14:09:10 2009 +0800
Change a debugging statement
diff --git a/lib/Prophet/Replica/prophet_cache.pm b/lib/Prophet/Replica/prophet_cache.pm
index 1ef4120..6bebe64 100644
--- a/lib/Prophet/Replica/prophet_cache.pm
+++ b/lib/Prophet/Replica/prophet_cache.pm
@@ -233,7 +233,7 @@ sub mirror_from {
-f File::Spec->catdir( $self->fs_root,
$self->changeset_cas->filename($key) ) );
- #warn "we don't have it at ".File::Spec->catdir( $self->fs_root, $self->changeset_cas->filename($key) );
+ #warn "Cache miss on ".File::Spec->catdir( $self->fs_root, $self->changeset_cas->filename($key) ."\n");
my $content = $source->_read_file( $source->changeset_cas->filename($key) );
utf8::decode($content) if utf8::is_utf8($content);
my $newkey = $self->changeset_cas->write(
commit 35db2ef4ee46a1511b840e02d8d4ceae422d1c5a
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 15:24:48 2009 +0800
perltidy
diff --git a/lib/Prophet/CLI/ProgressBar.pm b/lib/Prophet/CLI/ProgressBar.pm
index 7a2c169..1a139f5 100644
--- a/lib/Prophet/CLI/ProgressBar.pm
+++ b/lib/Prophet/CLI/ProgressBar.pm
@@ -14,8 +14,7 @@ sub progress_bar {
my $bar_count = 0;
my $format = $args{format};
return sub {
- print $bar->report( $format,
- , ++$bar_count );
+ print $bar->report( $format, ++$bar_count );
}
}
no Any::Moose;
commit aaa72b7b269123ae23bb40b202e826f87b86b74e
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 15:25:47 2009 +0800
make progress bars not explode when we're counting to 0
diff --git a/lib/Prophet/CLI/MirrorCommand.pm b/lib/Prophet/CLI/MirrorCommand.pm
index 7ae7289..1a1934a 100644
--- a/lib/Prophet/CLI/MirrorCommand.pm
+++ b/lib/Prophet/CLI/MirrorCommand.pm
@@ -25,12 +25,12 @@ sub sync_cache_from_source {
print "Mirroring resolutions from " . $args{source}->url . "\n";
$args{target}->resolution_db_handle->mirror_from(
source => $args{source}->resolution_db_handle,
- reporting_callback => $self->progress_bar( max => $args{source}->resolution_db_handle->latest_sequence_no )
+ reporting_callback => $self->progress_bar( max => ($args{source}->resolution_db_handle->latest_sequence_no ||0) )
);
print "\nMirroring changesets from " . $args{source}->url . "\n";
$args{target}->mirror_from(
source => $args{source},
- reporting_callback => $self->progress_bar( max => $args{source}->latest_sequence_no )
+ reporting_callback => $self->progress_bar( max => ($args{source}->latest_sequence_no ||0) )
);
}
commit 1d0966f8123f2de6bf3336071126257a0a876bf7
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 15:26:19 2009 +0800
It's ok if the replica we're reading from has no changesets
diff --git a/lib/Prophet/Replica/prophet_cache.pm b/lib/Prophet/Replica/prophet_cache.pm
index 6bebe64..33379b8 100644
--- a/lib/Prophet/Replica/prophet_cache.pm
+++ b/lib/Prophet/Replica/prophet_cache.pm
@@ -218,7 +218,7 @@ sub mirror_from {
if ( $source->can('read_changeset_index') ) {
$self->_write_file(
path => $self->changeset_index,
- content => ${ $source->read_changeset_index }
+ content => ${ $source->read_changeset_index ||''}
);
$self->traverse_changesets(
commit 4ac865b973c081a92b2cf627508991fdbee6dd6f
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 17:00:53 2009 +0800
Several fixes to properly initialize resolution databases
diff --git a/lib/Prophet/Replica/prophet_cache.pm b/lib/Prophet/Replica/prophet_cache.pm
index 33379b8..abe5419 100644
--- a/lib/Prophet/Replica/prophet_cache.pm
+++ b/lib/Prophet/Replica/prophet_cache.pm
@@ -55,12 +55,14 @@ has resdb_replica_uuid => (
has '+resolution_db_handle' => (
isa => 'Prophet::Replica | Undef',
lazy => 1,
+ weak_ref => 1,
default => sub {
my $self = shift;
- return if $self->is_resdb ;
+ return $self if $self->is_resdb ;
my $suffix = 'remote_replica_cache';
return Prophet::Replica->get_handle(
- { url => 'prophet_cache:'.$self->resdb_replica_uuid,
+ {
+ url => 'prophet_cache:'.$self->resdb_replica_uuid,
fs_root => File::Spec->catdir($self->app_handle->handle->resolution_db_handle->fs_root => $suffix),
app_handle => $self->app_handle,
db_uuid => $self->app_handle->handle->resolution_db_handle->db_uuid,
@@ -175,7 +177,7 @@ sub initialize {
mkpath( [ File::Spec->catdir( $self->fs_root => $_ ) ] );
}
- $self->set_db_uuid( $self->app_handle->handle->db_uuid);
+ $self->set_db_uuid( $args{db_uuid});
$self->set_resdb_replica_uuid($args{resdb_replica_uuid}) unless ($self->is_resdb);
$self->resolution_db_handle->initialize(db_uuid => $args{resdb_uuid}, replica_uuid => $args{resdb_replica_uuid}) unless ($self->is_resdb);
$self->after_initialize->($self);
@@ -204,23 +206,25 @@ sub replica_exists {
sub latest_sequence_no {
my $self = shift;
- my $count = (-s File::Spec->catdir($self->fs_root => $self->changeset_index )) / $self->CHG_RECORD_SIZE;
+ my $count = ((-s File::Spec->catdir($self->fs_root => $self->changeset_index )) ||0) / $self->CHG_RECORD_SIZE;
return $count;
}
sub mirror_from {
- my $self = shift;
- my %args = validate( @_, { source => 1, reporting_callback => {type => CODEREF, optional => 1 } });
+ my $self = shift;
+ my %args
+ = validate( @_, { source => 1, reporting_callback => { type => CODEREF, optional => 1 } } );
my $source = $args{source};
if ( $source->can('read_changeset_index') ) {
+ my $content = ${ $source->read_changeset_index } ||'';
+
$self->_write_file(
path => $self->changeset_index,
- content => ${ $source->read_changeset_index ||''}
+ content => $content
);
-
$self->traverse_changesets(
load_changesets => 0,
callback =>
@@ -228,21 +232,14 @@ sub mirror_from {
sub {
my $data = shift;
my ( $seq, $orig_uuid, $orig_seq, $key ) = @{$data};
- return
- if (
- -f File::Spec->catdir( $self->fs_root,
- $self->changeset_cas->filename($key) ) );
+ return if ( -f File::Spec->catdir( $self->fs_root, $self->changeset_cas->filename($key) ) );
#warn "Cache miss on ".File::Spec->catdir( $self->fs_root, $self->changeset_cas->filename($key) ."\n");
my $content = $source->_read_file( $source->changeset_cas->filename($key) );
utf8::decode($content) if utf8::is_utf8($content);
- my $newkey = $self->changeset_cas->write(
- $content
+ my $newkey = $self->changeset_cas->write( $content );
- );
-
- my $existsp = File::Spec->catdir( $self->fs_root,
- $self->changeset_cas->filename($key) );
+ my $existsp = File::Spec->catdir( $self->fs_root, $self->changeset_cas->filename($key) );
if ( !-f $existsp ) {
die "The mirror @{[$self->url]} appears to be incomplete. Perhaps a sync operation was aborted?\nCouldn't find changeset $key\n";
@@ -251,12 +248,12 @@ sub mirror_from {
,
after => 0,
- $args{reporting_callback} ? (reporting_callback => $args{reporting_callback}) : (),
+ $args{reporting_callback} ? ( reporting_callback => $args{reporting_callback} ) : (),
);
} else {
warn "Sorry, we only support replicas with a changeset index file";
}
- }
+}
1;
commit 3700b698f0c3879aa2dc0dc57650a3b9e70c704f
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 17:01:23 2009 +0800
Better error message when a replica does not exist
diff --git a/lib/Prophet/Replica.pm b/lib/Prophet/Replica.pm
index eca0477..10f2cd0 100644
--- a/lib/Prophet/Replica.pm
+++ b/lib/Prophet/Replica.pm
@@ -163,7 +163,7 @@ sub import_changesets {
my $source = $args{'from'};
- warn "The source does not exist" unless ($source->replica_exists);
+ warn "The source (@{[$source->url]}) does not exist" unless ($source->replica_exists);
$source->traverse_new_changesets(
for => $self,
commit 474629303b770961e07dd67da1e10cef3fad98df
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 17:01:54 2009 +0800
perltidy
diff --git a/lib/Prophet/CLI/Command/Clone.pm b/lib/Prophet/CLI/Command/Clone.pm
index 6aae8f8..feff57f 100644
--- a/lib/Prophet/CLI/Command/Clone.pm
+++ b/lib/Prophet/CLI/Command/Clone.pm
@@ -42,11 +42,7 @@ sub run {
}
$target->initialize(%init_args);
-
- $self->app_handle->config->set(
- _sources =>
- { $self->arg('from') => $self->arg('from') }
- );
+ $self->app_handle->config->set( _sources => { $self->arg('from') => $self->arg('from') });
$self->app_handle->config->save;
$self->SUPER::run();
commit 6a0f13e9979ad24725ab83ac60a7097e5922c12d
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 17:02:22 2009 +0800
Use new changeset cache when merging
diff --git a/lib/Prophet/CLI/Command/Merge.pm b/lib/Prophet/CLI/Command/Merge.pm
index 46d703f..44462a1 100644
--- a/lib/Prophet/CLI/Command/Merge.pm
+++ b/lib/Prophet/CLI/Command/Merge.pm
@@ -2,6 +2,7 @@ package Prophet::CLI::Command::Merge;
use Any::Moose;
extends 'Prophet::CLI::Command';
with 'Prophet::CLI::ProgressBar';
+with 'Prophet::CLI::MirrorCommand';
has source => ( isa => 'Prophet::Replica', is => 'rw');
has target => ( isa => 'Prophet::Replica', is => 'rw');
@@ -24,15 +25,16 @@ sub run {
return unless $self->validate_merge_replicas($self->source => $self->target);
-
-
-
-
+ if ( $self->source->can('read_changeset_index') && $self->target->url eq $self->app_handle->handle->url) {
+ my $original_source = $self->source;
+ $self->source($self->get_cache_for_source($original_source));
+ $self->sync_cache_from_source( target=> $self->source, source => $original_source);
+ }
$self->target->import_resolutions_from_remote_replica(
from => $self->source,
force => $self->has_arg('force'),
- );
-
+ ) if ($self->source->resolution_db_handle);
+
my $changesets = $self->_do_merge();
Prophet::CLI->start_pager();
commit 3397d8bfb085699a272a2d59ca53f20f3ef8b632
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 17:28:09 2009 +0800
Fix placeholder count
diff --git a/lib/Prophet/Replica/sqlite.pm b/lib/Prophet/Replica/sqlite.pm
index f6115ae..0aae52f 100644
--- a/lib/Prophet/Replica/sqlite.pm
+++ b/lib/Prophet/Replica/sqlite.pm
@@ -570,7 +570,7 @@ sub _write_changeset_to_db {
. "(creator, created,"
. "original_source_uuid, original_sequence_no, "
. "is_nullification, is_resolution, sha1) "
- . "VALUES(?,?,?,?,?,?)", {},
+ . "VALUES(?,?,?,?,?,?,?)", {},
$changeset->creator, $changeset->created,
$changeset->original_source_uuid,
commit b39f00a9f3e3f9bf6f951713816a9c5393b4a07d
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Thu Apr 16 17:28:33 2009 +0800
better caching diagnostics
diff --git a/lib/Prophet/Replica/prophet_cache.pm b/lib/Prophet/Replica/prophet_cache.pm
index abe5419..bdafaf5 100644
--- a/lib/Prophet/Replica/prophet_cache.pm
+++ b/lib/Prophet/Replica/prophet_cache.pm
@@ -183,7 +183,6 @@ sub initialize {
$self->after_initialize->($self);
}
-
sub set_resdb_replica_uuid {
my $self = shift;
my $id = shift;
@@ -193,7 +192,6 @@ sub set_resdb_replica_uuid {
);
}
-
sub replica_exists {
my $self = shift;
if (-e File::Spec->catdir($self->fs_root, $self->changeset_index)) {
@@ -210,8 +208,6 @@ sub latest_sequence_no {
return $count;
}
-
-
sub mirror_from {
my $self = shift;
my %args
@@ -232,17 +228,14 @@ sub mirror_from {
sub {
my $data = shift;
my ( $seq, $orig_uuid, $orig_seq, $key ) = @{$data};
- return if ( -f File::Spec->catdir( $self->fs_root, $self->changeset_cas->filename($key) ) );
-
- #warn "Cache miss on ".File::Spec->catdir( $self->fs_root, $self->changeset_cas->filename($key) ."\n");
+ if ( -e File::Spec->catdir( $self->fs_root, $self->changeset_cas->filename($key) ) ) {
+ return;
+ }
my $content = $source->_read_file( $source->changeset_cas->filename($key) );
utf8::decode($content) if utf8::is_utf8($content);
my $newkey = $self->changeset_cas->write( $content );
-
- my $existsp = File::Spec->catdir( $self->fs_root, $self->changeset_cas->filename($key) );
- if ( !-f $existsp ) {
- die "The mirror @{[$self->url]} appears to be incomplete. Perhaps a sync operation was aborted?\nCouldn't find changeset $key\n";
-
+ if ($newkey ne $key) {
+ Carp::confess "writing a mirrored changeset to the CAS resulted in an inconsistent hash. Corrupted upstream?";
}
}
@@ -255,5 +248,4 @@ sub mirror_from {
}
}
-
1;
-----------------------------------------------------------------------
More information about the Bps-public-commit
mailing list