[Bps-public-commit] r11618 - in SVN-PropDB: . lib/Prophet t
jesse at bestpractical.com
jesse at bestpractical.com
Mon Apr 7 09:15:58 EDT 2008
Author: jesse
Date: Mon Apr 7 09:15:37 2008
New Revision: 11618
Modified:
SVN-PropDB/ (props changed)
SVN-PropDB/lib/Prophet/Replica.pm
SVN-PropDB/lib/Prophet/Replica/HTTP.pm
SVN-PropDB/t/export.t
Log:
r29432 at 68-247-45-239: jesse | 2008-04-07 04:25:45 -0500
* replica doc, cleanup
Modified: SVN-PropDB/lib/Prophet/Replica.pm
==============================================================================
--- SVN-PropDB/lib/Prophet/Replica.pm (original)
+++ SVN-PropDB/lib/Prophet/Replica.pm Mon Apr 7 09:15:37 2008
@@ -416,41 +416,17 @@
publishing via HTTP or over a local filesystem for other Prophet replicas to clone or incorporate changes from.
-Inside the top level directory for the mirror, you'll find a directory named as B<a hex-encoded UUID>.
-This directory is the root of the published replica. The uuid uniquely identifes the database being replicated.
-All replicas of this database will share the same UUID.
-
-Inside the B<<db-uuid>> directory, are a set of files and directories that make up the actual content of the database replica:
-
-=over
-
-=item C<replica-uuid>
-
-=item C<latest-changeset>
-
-=item C<cas>
-
-=item C<records>
-
-=item C<changesets.idx>
-
-=item C<resolutions>
-
-=over
-
-=item
-
-
-=back
-
-=back
+=head3 text-dump replica format
+=head4 overview
$URL
/<db-uuid>/
/replica-uuid
- /latest
- /cas/<substr(sha1,0,1)>/substr(sha1,1,1)/<sha1>
+ /latest-sequence-no
+ /replica-version
+ /cas/records/<substr(sha1,0,1)>/substr(sha1,1,1)/<sha1>
+ /cas/changesets/<substr(sha1,0,1)>/substr(sha1,1,1)/<sha1>
/records (optional?)
/<record type> (for resolution is actually _prophet-resolution-<cas-key>)
/<record uuid> which is a file containing a list of 0 or more rows
@@ -464,7 +440,7 @@
/resolutions/
/replica-uuid
- /latest
+ /latest-sequence-no
/cas/<substr(sha1,0,1)>/substr(sha1,1,1)/<sha1>
/content (optional?)
/_prophet-resolution-<cas-key> (cas-key == a hash the conflicting change)
@@ -476,6 +452,81 @@
each record is : local-replica-seq-no : original-uuid : original-seq-no : cas key
...
+
+Inside the top level directory for the mirror, you'll find a directory named as B<a hex-encoded UUID>.
+This directory is the root of the published replica. The uuid uniquely identifes the database being replicated.
+All replicas of this database will share the same UUID.
+
+Inside the B<<db-uuid>> directory, are a set of files and directories that make up the actual content of the database replica:
+
+=over
+
+=item C<replica-uuid>
+
+Contains the replica's hex-encoded UUID.
+
+=item C<replica-version>
+
+Contains a single integer that defines the replica format.
+
+The current replica version is 1.
+
+=item C<latest-sequence-no>
+
+Contains a single integer, the replica's most recent sequence number.
+
+=item C<cas>
+
+The C<cas> directory holds changesets and records, each keyed by a
+hex-encoded hash of the item's content. Inside the C<cas> directory, you'll find
+a two-level deep directory tree of single-character hex digits.
+You'll find the content with the key C<f4b7489b21f8d107ad8df78750a410c028abbf6c>
+inside C<cas/f/4>
+
+
+
+
+=item C<records>
+
+Files inside the C<records> directory are index files which list off all published versions of a record and the key necessary to retrieve the record from the I<content-addressed store>.
+
+Inside the C<records> directory, you'll find directories named for each
+C<type> in your database. Inside each C<type> directory, you'll find a two-level directory tree of single hexadecimal digits. You'll find the record with the type <Foo> and the UUID C<29A3CA16-03C5-11DD-9AE0-E25CFCEE7EC4> stored in
+
+ records/Foo/2/9/29A3CA16-03C5-11DD-9AE0-E25CFCEE7EC4
+
+
+The format of record files is:
+
+ <unsigned-long-int: last-changed-sequence-no><40 chars of hex: cas key>
+
+The file is sorted in asecnding order by revision id.
+
+
+=item C<changesets.idx>
+
+The C<changesets.idx> file lists each changeset in this replica and
+provides an index into the B<content-addressed storage> to fetch
+the content of the changeset.
+
+The format of record files is:
+
+ <unsigned-long-int: sequence-no><16 bytes: changeset original source uuid><unsigned-long-int: changeset original source sequence no><16 bytes: cas key - sha1 sum of the changeset's content>
+
+The file is sorted in ascending order by revision id.
+
+
+=item C<resolutions>
+
+=over
+
+=item
+
+
+=back
+
+=back
+
=cut
sub export_to {
@@ -484,12 +535,16 @@
my $replica_root = dir( $path, $self->db_uuid );
my $cas_dir = dir( $replica_root => 'cas' );
+ my $record_cas_dir = dir( $cas_dir => 'records' );
+ my $changeset_cas_dir = dir( $cas_dir => 'changesets' );
my $record_dir = dir( $replica_root => 'records' );
_mkdir($path);
_mkdir($replica_root);
_mkdir($record_dir);
- make_tiered_dirs($cas_dir);
+ _mkdir($cas_dir);
+ make_tiered_dirs($record_cas_dir);
+ make_tiered_dirs($changeset_cas_dir);
$self->_init_export_metadata( root => $replica_root );
@@ -497,11 +552,11 @@
$self->export_records(
type => $type,
root => $replica_root,
- cas_dir => $cas_dir
+ cas_dir => $record_cas_dir
);
}
- $self->export_changesets( root => $replica_root, cas_dir => $cas_dir );
+ $self->export_changesets( root => $replica_root, cas_dir => $changeset_cas_dir );
}
sub _init_export_metadata {
@@ -509,8 +564,8 @@
my %args = validate( @_, { root => 1 } );
$self->_output_oneliner_file( path => file( $args{'root'}, 'replica-uuid' ), content => $self->uuid );
- $self->_output_oneliner_file( path => file( $args{'root'}, 'latest' ), content => $self->most_recent_changeset );
- $self->_output_oneliner_file( path => file( $args{'root'}, 'repository-version' ), content => '1' );
+ $self->_output_oneliner_file( path => file( $args{'root'}, 'replica-version' ), content => '1' );
+ $self->_output_oneliner_file( path => file( $args{'root'}, 'latest-sequence-no' ), content => $self->most_recent_changeset );
}
@@ -562,7 +617,7 @@
# XXX TODO FETCH THAT
my $record_last_changed_changeset = 1;
- my $index_row = pack( 'Na16H40', $record_last_changed_changeset, $args{record}->uuid, $cas_key );
+ my $index_row = pack( 'NH40', $record_last_changed_changeset, $cas_key );
print $record_index $index_row || die $!;
close $record_index;
}
Modified: SVN-PropDB/lib/Prophet/Replica/HTTP.pm
==============================================================================
--- SVN-PropDB/lib/Prophet/Replica/HTTP.pm (original)
+++ SVN-PropDB/lib/Prophet/Replica/HTTP.pm Mon Apr 7 09:15:37 2008
@@ -67,7 +67,7 @@
my $first_rev = ( $args{'after'} + 1 ) || 1;
- my $latest = LWP::Simple::get($self->url.'/latest');
+ my $latest = LWP::Simple::get($self->url.'/latest-sequence-no');
my $chgidx = LWP::Simple::get($self->url.'/changesets.idx');
@@ -77,7 +77,7 @@
$orig_uuid = Data::UUID->new->to_string( $orig_uuid );
# XXX: deserialize the changeset content from the cas with $key
- my $casfile = $self->url.'/cas/'.substr($key, 0, 1).'/'.substr($key, 1, 1).'/'.$key;
+ my $casfile = $self->url.'/cas/changesets/'.substr($key, 0, 1).'/'.substr($key, 1, 1).'/'.$key;
my $content = YAML::Syck::Load(LWP::Simple::get($casfile));
Modified: SVN-PropDB/t/export.t
==============================================================================
--- SVN-PropDB/t/export.t (original)
+++ SVN-PropDB/t/export.t Mon Apr 7 09:15:37 2008
@@ -52,7 +52,7 @@
};
ok( -e $path->file('changesets.idx'), 'found changesets index' );
- my $latest = $path->file('latest')->slurp;
+ my $latest = $path->file('latest-sequence-no')->slurp;
is($latest, 5);
use_ok('Prophet::Replica::HTTP');
my $changesets = Prophet::Replica->new({ url => 'prophet:file://'.$path} )->fetch_changesets( after => 0 );
More information about the Bps-public-commit
mailing list