[svk-commit] r3000 - in trunk: . lib/SVK/Command lib/SVK/Mirror/Backend t/mirror utils
nobody at bestpractical.com
nobody at bestpractical.com
Fri Jul 18 19:01:59 EDT 2008
Author: clkao
Date: Fri Jul 18 19:01:58 2008
New Revision: 3000
Added:
trunk/t/mirror/sync-bootstrap.t
trunk/utils/mipush (contents, props changed)
Modified:
trunk/ (props changed)
trunk/MANIFEST
trunk/Makefile.PL
trunk/lib/SVK/Command/Mirror.pm
trunk/lib/SVK/Mirror.pm
trunk/lib/SVK/Mirror/Backend/SVNRa.pm
trunk/lib/SVK/Notify.pm
Log:
- Merge //mirror/svk/branches/mirror-boostrap to //mirror/svk/trunk
Modified: trunk/MANIFEST
==============================================================================
--- trunk/MANIFEST (original)
+++ trunk/MANIFEST Fri Jul 18 19:01:58 2008
@@ -303,6 +303,7 @@
t/mirror/commit-copy.t
t/mirror/dav-authz.t
t/mirror/relocate.t
+t/mirror/sync-bootstrap.t
t/mirror/sync-crazy-replace.t
t/mirror/sync-empty.t
t/mirror/sync-escape.t
Modified: trunk/Makefile.PL
==============================================================================
--- trunk/Makefile.PL (original)
+++ trunk/Makefile.PL Fri Jul 18 19:01:58 2008
@@ -91,6 +91,8 @@
'Pod::Simple' => '0', # in core since 5.9.3
'File::Spec' => '3.19', # in core since 5.9.3
);
+
+
requires(
'Date::Format' => '',
) if ($^O eq 'MSWin32');
@@ -98,6 +100,13 @@
'Test::More' => '0.42',
);
features(
+ 'Bootstrap support' => [
+ -default => 1,
+ 'PerlIO::via::Bzip2' => '0',
+ 'PerlIO::gzip' => '0',
+ 'Term::ProgressBar' => '0',
+ 'SVN::Dump' => '0.04',
+ ],
'Localized messages' => [
-default => 1,
'Locale::Maketext::Lexicon' => '0.62',
@@ -139,7 +148,7 @@
auto_install();
auto_provides();
-WriteAll( sign => 1 );
+WriteAll( sign => 0 );
$::VERSION = $required_svn;
die << "." unless eval {require SVN::Core; SVN::Core->VERSION >= main->VERSION };
Modified: trunk/lib/SVK/Command/Mirror.pm
==============================================================================
--- trunk/lib/SVK/Command/Mirror.pm (original)
+++ trunk/lib/SVK/Command/Mirror.pm Fri Jul 18 19:01:58 2008
@@ -62,6 +62,7 @@
sub options {
('l|list' => 'list',
'd|delete|detach'=> 'detach',
+ 'b|bootstrap=s' => 'bootstrap',
'upgrade' => 'upgrade',
'relocate'=> 'relocate',
'unlock'=> 'unlock',
@@ -146,6 +147,87 @@
return;
}
+package SVK::Command::Mirror::bootstrap;
+use base qw(SVK::Command::Mirror);
+use SVK::I18N;
+use SVK::Logger;
+
+use constant narg => 2;
+
+sub run {
+ my ($self, $target, $uri, @options) = @_;
+ my ($m, $mpath) = $target->is_mirrored;
+
+ die loc("No such dump file: %1.\n", $self->{bootstrap})
+ unless $self->{bootstrap} eq '-' ||
+ $self->{bootstrap} eq 'auto' || -f ($self->{bootstrap});
+
+ if (!$m) {
+ $self->SUPER::run($target,$uri, @options);
+ ($m, $mpath) = $target->is_mirrored;
+ }
+ # XXX: make sure the mirror is fresh and not synced at all
+
+ die loc("%1 is not a mirrored path.\n", $target->depotpath) if !$m;
+ die loc("%1 is inside a mirrored path.\n", $target->depotpath) if $mpath;
+
+ my $hint;
+ if ( $self->{bootstrap} eq 'auto' ) {
+ my $ra = $m->_backend->_new_ra;
+ $ra->reparent( $ra->get_repos_root() );
+ my %prop = %{ ( $ra->get_file( '', $ra->get_latest_revnum, undef ) )[1] };
+ $m->_backend->_ra_finished($ra);
+ if ( $hint = $prop{'svk:dump-url'} ) {
+ $logger->info( loc( "Downloading dump file: %1", $hint ) );
+ $self->{bootstrap} = File::Temp->new;
+
+ require LWP::UserAgent;
+ my $ua = LWP::UserAgent->new;
+ my ( $received_size, $next_update ) = ( 0, 0 );
+ my $did_set_target = 0;
+ # XXX: switch to a default notify object that takes care
+ # of quiet and gui variants.
+ my $progress = SVK::Notify->new->progress(
+ min => 0,
+ max => 1024,
+ );
+ $ua->get(
+ $hint,
+ ':content_cb' => sub {
+ my ( $data, $cb_response, $protocol ) = @_;
+
+ if ($progress) {
+ unless ($did_set_target) {
+ if ( my $content_length = $cb_response->content_length ) {
+ $progress->attr(max => $content_length);
+ $did_set_target = 1;
+ }
+ else {
+ $progress->attr(max =>
+ $received_size + 2 * length $data );
+ }
+ }
+ }
+ $received_size += length $data;
+ print { $self->{bootstrap} } $data;
+ if ($progress && $received_size >= $next_update) {
+ local $| = 1;
+ print STDERR $progress->report( "%45b %p\r", $received_size );
+ $next_update = $received_size + 1;
+ }
+ },
+ ':read_size_hint' => 16384,
+ );
+ }
+
+ }
+
+ $logger->info( loc("Bootstrapping mirror from dump") );
+ $m->bootstrap($self->{bootstrap}, $hint); # load from dumpfile
+ print loc("Mirror path '%1' synced from dumpfile.\n", $target->depotpath);
+ return;
+}
+
package SVK::Command::Mirror::upgrade;
use base qw(SVK::Command::Mirror);
use SVK::I18N;
@@ -336,6 +418,7 @@
# You may also list the target part first:
mirror DEPOTPATH [http|svn]://host/path
+ mirror --bootstrap=DUMPFILE DEPOTPATH [http|svn]://host/path
mirror --list [DEPOTNAME...]
mirror --relocate DEPOTPATH [http|svn]://host/path
mirror --detach DEPOTPATH
@@ -346,6 +429,7 @@
=head1 OPTIONS
+ -b [--bootstrap] : mirror from a dump file
-l [--list] : list mirrored paths
-d [--detach] : mark a depotpath as no longer mirrored
--relocate : change the upstream URI for the mirrored depotpath
Modified: trunk/lib/SVK/Mirror.pm
==============================================================================
--- trunk/lib/SVK/Mirror.pm (original)
+++ trunk/lib/SVK/Mirror.pm Fri Jul 18 19:01:58 2008
@@ -232,6 +232,110 @@
$editor->close_edit;
}
+=item bootstrap
+
+=cut
+
+sub bootstrap {
+ my ($self, $dumpfile, $file_hint) = @_;
+ $file_hint ||= $dumpfile;
+ my $fh;
+ require SVN::Dump;
+
+ if ($dumpfile eq '-') {
+ $fh = \*STDIN;
+ }
+ else {
+ open $fh, '<', $dumpfile or die $!;
+ }
+
+ # XXX make these fail optionally
+ if ($file_hint =~ m/bz2$/i) {
+ require PerlIO::via::Bzip2;
+ binmode($fh, ':via(Bzip2)');
+ }
+ elsif ($file_hint =~ m/gz$/i) {
+ require PerlIO::gzip;
+ binmode($fh, ':gzip(lazy)');
+ }
+
+ my $dump = SVN::Dump->new( { fh => $fh } );
+ my $prefix = $self->path.'/';
+
+ my $prev = undef;
+ my $rev = 0;
+ my $buf;
+ my $header;
+ my $progress = SVK::Notify->new->progress( min => 0, max => $self->_backend->_new_ra->get_latest_revnum );
+ if ($self->fromrev) {
+ $logger->info(loc("Skipping dumpstream up to revision %1", $self->fromrev));
+ }
+ while ( my $record = $dump->next_record() ) {
+ if ($record->type eq 'format' || $record->type eq 'uuid') {
+ $header = $header.$record->as_string;
+ next;
+ }
+
+ my $translate = sub {
+ my $rec = shift;
+
+ if (my $path = $rec->get_header('Node-copyfrom-path')) {
+ $path = $prefix.$path;
+ $rec->set_header('Node-copyfrom-path' => $path );
+ }
+ if (my $rev = $rec->get_header('Node-copyfrom-rev')) {
+# $rec->set_header('Node-copyfrom-rev' =>
+# scalar $self->find_local_rev( $rev, $self->source_uuid ) - 1);
+ }
+
+ if ($rec->get_header('Revision-number')) {
+ $| = 1;
+ $rev = $rec->get_header('Revision-number');
+ $prev = $rev if !$prev;
+ $rec->set_property('svm:headrev',$self->source_uuid.':'.$rev."\n");
+ printf STDERR "%s rev:%d\r",$progress->report( "%45b",$rev),$rev;
+ }
+
+
+ if ( my $path = $rec->get_header('Node-path') ) {
+ $path = $prefix.$path;
+ $rec->set_header('Node-path' => $path);
+ }
+
+ };
+ $translate->( $record );
+ my $inc = $record->get_included_record;
+ $translate->( $inc ) if $inc;
+
+ if ($rev and $prev != $rev) {
+ $self->_import_repos($header,$buf) if $prev > $self->fromrev;
+ $buf = "";
+ $prev = $rev;
+ }
+
+ $buf = $buf.$record->as_string;
+ }
+ # last one
+ if ($rev) {
+ $self->_import_repos($header, $buf) if $prev > $self->fromrev;
+ }
+
+}
+
+sub _import_repos {
+ my $self = shift;
+ my ($header, $buf) = @_;
+ $buf = $header.$buf;
+ open my $fh, '<', \$buf;
+ my $feedback = '';
+ open my $fstream, '>', \$feedback;
+ my $ret = SVN::Repos::load_fs2( $self->repos, $fh, $fstream, $SVN::Repos::load_uuid_default, undef, 0, 0, undef, undef );
+ # (repos,dumpstream,feedback_stream,uuid_action,parent_dir,use_pre_commit_hook,use_post_commit_hook,cancel_func,cancel_baton,pool);
+ # XXX: display $feedback if we are in verbose / debug mode.
+ # and provide progress feedback in caller
+ return $ret;
+}
+
=item relocate($newurl)
=item with_lock($code)
@@ -369,10 +473,13 @@
$logger->info(loc("Syncing %1", $self->url).($self->_backend->_relayed ? loc(" via %1", $self->server_url) : ""));
+ $self->{use_progress} = 1 unless SVK::Test->can('is_output');
+
$self->mirror_changesets($torev,
sub {
my ( $changeset, $rev ) = @_;
- $logger->info("Committed revision $rev from revision $changeset.");
+ $logger->info("Committed revision $rev from revision $changeset.")
+ unless $self->{use_progress};
}, $fake_last
);
die $@ if $@;
Modified: trunk/lib/SVK/Mirror/Backend/SVNRa.pm
==============================================================================
--- trunk/lib/SVK/Mirror/Backend/SVNRa.pm (original)
+++ trunk/lib/SVK/Mirror/Backend/SVNRa.pm Fri Jul 18 19:01:58 2008
@@ -779,7 +779,12 @@
}
$ra = SVK::Mirror::Backend::SVNRaPipe->new( $ra, sub { shift @gen } );
}
+ my $progress =
+ $self->mirror->{use_progress}
+ ? SVK::Notify->new->progress( max => scalar @$revs )
+ : undef;
my $pool = SVN::Pool->new_default;
+ my $i = 0;
for (@$revs) {
$pool->clear;
my ( $changeset, $metadata ) = @$_;
@@ -797,6 +802,8 @@
$ra->replay( $changeset, 0, 1, $editor );
$self->_after_replay($ra, $editor);
}, $translate_from );
+ local $| = 1;
+ print STDERR $progress->report( "%45b %p\r", ++$i ) if $progress;
}
$self->_ra_finished($ra);
}
Modified: trunk/lib/SVK/Notify.pm
==============================================================================
--- trunk/lib/SVK/Notify.pm (original)
+++ trunk/lib/SVK/Notify.pm Fri Jul 18 19:01:58 2008
@@ -209,5 +209,14 @@
$self->flush ($path, 1);
}
+sub progress {
+ my $self = shift;
+ require Time::Progress;
+ return if $self->{quiet};
+ my $progress = Time::Progress->new();
+ $progress->attr (@_);
+ return $progress;
+
+}
1;
Added: trunk/t/mirror/sync-bootstrap.t
==============================================================================
--- (empty file)
+++ trunk/t/mirror/sync-bootstrap.t Fri Jul 18 19:01:58 2008
@@ -0,0 +1,75 @@
+#!/usr/bin/perl -w
+use strict;
+use Test::More;
+use SVK::Test;
+eval { require SVN::Mirror; 1 } or plan skip_all => 'require SVN::Mirror';
+plan tests => 8;
+
+my ($xd, $svk) = build_test('test', 'm2', 'm3');
+
+our $output;
+
+my $tree = create_basic_tree($xd, '/test/');
+my $depot = $xd->find_depot('test');
+
+my $uri = uri($depot->repospath);
+
+my $dump = File::Temp->new;
+dump_all($depot => $dump);
+close $dump;
+
+is_output($svk, mirror => ['//m', $uri],
+ ["Mirror initialized. Run svk sync //m to start mirroring."]);
+
+is_output($svk, mirror => ['/m2/m', $uri],
+ ["Mirror initialized. Run svk sync /m2/m to start mirroring."]);
+$svk->sync('/m2/m');
+
+is_output($svk, mirror => ['--bootstrap='.$dump, '//m', $uri],
+ ['Bootstrapping mirror from dump',
+ 'Mirror path \'//m\' synced from dumpfile.']);
+
+# compare normal mirror result and bootstrap mirror result
+my ($exp_mirror, $boot_mirror);
+open my $exp, '>', \$exp_mirror;
+open my $boot, '>', \$boot_mirror;
+dump_all($xd->find_depot('') => $boot);
+dump_all($xd->find_depot('m2') => $exp);
+$exp_mirror =~ s/UUID: .*//;
+$boot_mirror =~ s/UUID: .*//;
+# remove first svn-date (initial mirro)
+# 2007-08-09T14:43:18.137165Z
+$exp_mirror =~ s/\d{4}-\d{2}-\d{2}T[\d:.]+Z//;
+$boot_mirror =~ s/\d{4}-\d{2}-\d{2}T[\d:.]+Z//;
+
+is($boot_mirror, $exp_mirror, 'UUID should be the same'); # do something with UUID, they should be identical
+
+# now try with mirror, sync in single bootstrap command
+
+# try feed with incorrect file
+is_output($svk, mirror => ['--bootstrap=./no-such-file', '/m3/m', $uri],
+ ["No such dump file: ./no-such-file."]);
+# this is real test
+is_output($svk, mirror => ['--bootstrap',$dump->filename, '/m3/m', $uri],
+ ['Mirror initialized. Run svk sync /m3/m to start mirroring.',
+ 'Bootstrapping mirror from dump',
+ "Mirror path '/m3/m' synced from dumpfile."]);
+
+# compare UUID
+
+my ($boot_mirror2);
+open my $boot2, '>', \$boot_mirror2;
+dump_all($xd->find_depot('m3') => $boot2);
+$boot_mirror2 =~ s/UUID: .*//;
+# remove first svn-date (initial mirro)
+# 2007-08-09T14:43:18.137165Z
+$boot_mirror2 =~ s/\d{4}-\d{2}-\d{2}T[\d:.]+Z//;
+
+is($boot_mirror, $boot_mirror2, 'UUID should be the same');
+is($exp_mirror, $boot_mirror2, 'UUID should be the same');
+
+sub dump_all {
+ my ($depot, $output) = @_;
+ my $repos = $depot->repos;
+ SVN::Repos::dump_fs2($repos, $output, undef, 1, $repos->fs->youngest_rev, 0, 0, undef, undef);
+}
Added: trunk/utils/mipush
==============================================================================
--- (empty file)
+++ trunk/utils/mipush Fri Jul 18 19:01:58 2008
@@ -0,0 +1,110 @@
+#!/usr/bin/perl -w
+use strict;
+use SVK;
+use SVK::XD;
+use SVN::Repos;
+use SVK::Util 'traverse_history';
+use Getopt::Long;
+#15:03 <clkao> mirror boot
+#15:03 <clkao> o mirror state initialisation, SVK::Mirror accessible
+#15:03 <clkao> o dump->read_record
+#15:03 <clkao> o translate record, add revprop for svm:head
+#15:03 <clkao> o svn::repos::load_from_dump($record->as_string)
+
+my $revspec;
+
+sub usage {
+ print <<EOUSAGE;
+Usage: mipush repopath depotpath svn_dump_file
+ Reverse version of pullyu
+
+ Example: ./mipush ./.svk/local //mirror/svk myproject-svn-dumpfile
+
+EOUSAGE
+exit;
+}
+
+
+die unless GetOptions ("r|revision=s@" => \$revspec);
+
+use SVN::Dump 0.03;
+
+my $repospath = shift or usage();
+my $depotpath = shift or usage();
+my ($depotname, $path) = $depotpath =~ m|^/([^/]*)(/.*?)/?$|;
+my $file = shift or usage();
+
+my $repos = SVN::Repos::open($repospath) or die $!;
+my $depot = SVK::Depot->new({ depotname => $depotname, repos => $repos, repospath => $repospath});
+# TODO: for exists mirror, maybe we should remove it and create a new one?
+my $t = SVK::Path->real_new({ depot => $depot, path => $path })
+ ->refresh_revision;
+my ( $m, $mpath ) = $t->is_mirrored;
+die $t->depotpath." is not a mirrored path.\n" if !$m;
+die $t->depotpath." is inside a mirrored path.\n" if $mpath;
+die "only whole repository mirrors are supported.\n" if length($mpath);
+
+
+my $prev = undef;
+my $rev = 0;
+my $dump = SVN::Dump->new( { file => $file } );
+my $prefix = $m->path.'/';
+my $buf;
+my $header;
+while ( my $record = $dump->next_record() ) {
+ if ($record->type eq 'format' || $record->type eq 'uuid') {
+ $header = $header.$record->as_string;
+ next;
+ }
+
+ my $translate = sub {
+ my $rec = shift;
+
+ if (my $path = $rec->get_header('Node-copyfrom-path')) {
+ $path = $prefix.$path;
+ $rec->set_header('Node-copyfrom-path' => $path );
+ }
+# if (my $rev = $rec->get_header('Node-copyfrom-rev')) {
+# $rec->set_header('Node-copyfrom-rev' => 0);
+# scalar $m->find_local_rev( $rev, $m->source_uuid ) - $rev);
+# warn "$rev changed to ". (scalar $m->find_local_rev( $rev, $m->source_uuid ) - $rev) ;
+# }
+
+ if ($rec->get_header('Revision-number')) {
+ $rev = $rec->get_header('Revision-number');
+ $prev = $rev if !$prev;
+ $rec->set_property('svm:headrev',$m->source_uuid.':'.$rev."\n");
+ }
+
+
+ if ( my $path = $rec->get_header('Node-path') ) {
+ $path = $prefix.$path;
+ $rec->set_header('Node-path' => $path);
+ }
+
+ };
+ $translate->( $record );
+ my $inc = $record->get_included_record;
+ $translate->( $inc ) if $inc;
+
+ if ($rev and $prev != $rev) {
+ import_repos($header,$buf);
+ $buf = "";
+ $prev = $rev;
+ }
+
+ $buf = $buf.$record->as_string;
+}
+
+if ($rev) {
+ import_repos($header,$buf);
+}
+sub import_repos {
+ my ($header, $buf) = @_;
+ $buf = $header.$buf;
+ open my $fh, '<', \$buf;
+ my $ret = SVN::Repos::load_fs2( $repos, $fh, \*STDERR, $SVN::Repos::load_uuid_default, undef, 0, 0, undef, undef );
+ # (repos,dumpstream,feedback_stream,uuid_action,parent_dir,use_pre_commit_hook,use_post_commit_hook,cancel_func,cancel_baton,pool);
+ return $ret;
+}
+
More information about the svk-commit
mailing list