[svk-commit] r3000 - in trunk: . lib/SVK/Command lib/SVK/Mirror/Backend t/mirror utils

nobody at bestpractical.com nobody at bestpractical.com
Fri Jul 18 19:01:59 EDT 2008


Author: clkao
Date: Fri Jul 18 19:01:58 2008
New Revision: 3000

Added:
   trunk/t/mirror/sync-bootstrap.t
   trunk/utils/mipush   (contents, props changed)
Modified:
   trunk/   (props changed)
   trunk/MANIFEST
   trunk/Makefile.PL
   trunk/lib/SVK/Command/Mirror.pm
   trunk/lib/SVK/Mirror.pm
   trunk/lib/SVK/Mirror/Backend/SVNRa.pm
   trunk/lib/SVK/Notify.pm

Log:
- Merge //mirror/svk/branches/mirror-boostrap to //mirror/svk/trunk

Modified: trunk/MANIFEST
==============================================================================
--- trunk/MANIFEST	(original)
+++ trunk/MANIFEST	Fri Jul 18 19:01:58 2008
@@ -303,6 +303,7 @@
 t/mirror/commit-copy.t
 t/mirror/dav-authz.t
 t/mirror/relocate.t
+t/mirror/sync-bootstrap.t
 t/mirror/sync-crazy-replace.t
 t/mirror/sync-empty.t
 t/mirror/sync-escape.t

Modified: trunk/Makefile.PL
==============================================================================
--- trunk/Makefile.PL	(original)
+++ trunk/Makefile.PL	Fri Jul 18 19:01:58 2008
@@ -91,6 +91,8 @@
     'Pod::Simple'              => '0',       # in core since 5.9.3
     'File::Spec'               => '3.19',    # in core since 5.9.3
 );
+
+
 requires(
     'Date::Format'             => '',
 ) if ($^O eq 'MSWin32');
@@ -98,6 +100,13 @@
     'Test::More'               => '0.42',
 );
 features(
+    'Bootstrap support' => [
+    -default => 1,
+        'PerlIO::via::Bzip2'       => '0',
+        'PerlIO::gzip'             => '0',
+        'Term::ProgressBar'        => '0',
+        'SVN::Dump'                => '0.04',
+    ],
     'Localized messages' => [
 	-default => 1,
         'Locale::Maketext::Lexicon' => '0.62',
@@ -139,7 +148,7 @@
 auto_install();
 
 auto_provides();
-WriteAll( sign => 1 );
+WriteAll( sign => 0 );
 
 $::VERSION = $required_svn;
 die << "." unless eval {require SVN::Core; SVN::Core->VERSION >= main->VERSION };

Modified: trunk/lib/SVK/Command/Mirror.pm
==============================================================================
--- trunk/lib/SVK/Command/Mirror.pm	(original)
+++ trunk/lib/SVK/Command/Mirror.pm	Fri Jul 18 19:01:58 2008
@@ -62,6 +62,7 @@
 sub options {
     ('l|list'  => 'list',
      'd|delete|detach'=> 'detach',
+     'b|bootstrap=s' => 'bootstrap',
      'upgrade' => 'upgrade',
      'relocate'=> 'relocate',
      'unlock'=> 'unlock',
@@ -146,6 +147,87 @@
     return;
 }
 
+package SVK::Command::Mirror::bootstrap;
+use base qw(SVK::Command::Mirror);
+use SVK::I18N;
+use SVK::Logger;
+
+use constant narg => 2;
+
+sub run {
+    my ($self, $target, $uri, @options) = @_;
+    my ($m, $mpath) = $target->is_mirrored;
+
+    die loc("No such dump file: %1.\n", $self->{bootstrap})
+        unless $self->{bootstrap} eq '-' ||
+	$self->{bootstrap} eq 'auto' || -f ($self->{bootstrap});
+
+    if (!$m) {
+	$self->SUPER::run($target,$uri, @options);
+	($m, $mpath) = $target->is_mirrored;
+    }
+    # XXX: make sure the mirror is fresh and not synced at all
+
+    die loc("%1 is not a mirrored path.\n", $target->depotpath) if !$m;
+    die loc("%1 is inside a mirrored path.\n", $target->depotpath) if $mpath;
+
+    my $hint;
+    if ( $self->{bootstrap} eq 'auto' ) {
+        my $ra = $m->_backend->_new_ra;
+        $ra->reparent( $ra->get_repos_root() );
+        my %prop = %{ ( $ra->get_file( '', $ra->get_latest_revnum, undef ) )[1] };
+        $m->_backend->_ra_finished($ra);
+        if ( $hint = $prop{'svk:dump-url'} ) {
+            $logger->info( loc( "Downloading dump file: %1", $hint ) );
+            $self->{bootstrap} = File::Temp->new;
+
+            require LWP::UserAgent;
+            my $ua = LWP::UserAgent->new;
+            my ( $received_size, $next_update ) = ( 0, 0 );
+            my $did_set_target = 0;
+            # XXX: switch to a default notify object that takes care
+            # of quiet and gui variants.
+            my $progress = SVK::Notify->new->progress(
+                    min => 0,
+                    max => 1024,
+            );
+            $ua->get(
+                $hint,
+                ':content_cb' => sub {
+                    my ( $data, $cb_response, $protocol ) = @_;
+
+                    if ($progress) {
+                        unless ($did_set_target) {
+                            if ( my $content_length = $cb_response->content_length ) {
+                                $progress->attr(max => $content_length);
+                                $did_set_target = 1;
+                            }
+                            else {
+                                $progress->attr(max => 
+                                                  $received_size + 2 * length $data );
+                            }
+                        }
+                    }
+                    $received_size += length $data;
+                    print { $self->{bootstrap} } $data;
+		    if ($progress && $received_size >= $next_update) {
+			local $| = 1;
+			print STDERR $progress->report( "%45b %p\r", $received_size );
+			$next_update = $received_size + 1;
+		    }
+                },
+                ':read_size_hint' => 16384,
+            );
+        }
+
+    }
+
+    $logger->info( loc("Bootstrapping mirror from dump") );
+    $m->bootstrap($self->{bootstrap}, $hint); # load from dumpfile
+    print loc("Mirror path '%1' synced from dumpfile.\n", $target->depotpath);
+    return;
+}
+
 package SVK::Command::Mirror::upgrade;
 use base qw(SVK::Command::Mirror);
 use SVK::I18N;
@@ -336,6 +418,7 @@
  # You may also list the target part first:
  mirror DEPOTPATH [http|svn]://host/path
 
+ mirror --bootstrap=DUMPFILE DEPOTPATH [http|svn]://host/path 
  mirror --list [DEPOTNAME...]
  mirror --relocate DEPOTPATH [http|svn]://host/path 
  mirror --detach DEPOTPATH
@@ -346,6 +429,7 @@
 
 =head1 OPTIONS
 
+ -b [--bootstrap]       : mirror from a dump file
  -l [--list]            : list mirrored paths
  -d [--detach]          : mark a depotpath as no longer mirrored
  --relocate             : change the upstream URI for the mirrored depotpath

Modified: trunk/lib/SVK/Mirror.pm
==============================================================================
--- trunk/lib/SVK/Mirror.pm	(original)
+++ trunk/lib/SVK/Mirror.pm	Fri Jul 18 19:01:58 2008
@@ -232,6 +232,110 @@
     $editor->close_edit;
 }
 
+=item bootstrap
+
+=cut
+
+sub bootstrap {
+    my ($self, $dumpfile, $file_hint) = @_;
+    $file_hint ||= $dumpfile;
+    my $fh;
+    require SVN::Dump;
+
+    if ($dumpfile eq '-') {
+        $fh = \*STDIN;
+    }
+    else {
+        open $fh, '<', $dumpfile or die $!;
+    }
+
+    # XXX make these fail optionally
+    if ($file_hint =~ m/bz2$/i) {
+        require PerlIO::via::Bzip2;
+        binmode($fh, ':via(Bzip2)');
+    }
+    elsif ($file_hint =~ m/gz$/i) {
+        require PerlIO::gzip;
+        binmode($fh, ':gzip(lazy)');
+    }
+
+    my $dump = SVN::Dump->new( { fh => $fh } );
+    my $prefix = $self->path.'/';
+
+    my $prev = undef;
+    my $rev = 0;
+    my $buf;
+    my $header;
+    my $progress = SVK::Notify->new->progress( min => 0, max => $self->_backend->_new_ra->get_latest_revnum );
+    if ($self->fromrev) {
+        $logger->info(loc("Skipping dumpstream up to revision %1", $self->fromrev));
+    }
+    while ( my $record = $dump->next_record() ) {
+	if ($record->type eq 'format' || $record->type eq 'uuid') {
+	    $header = $header.$record->as_string;
+	    next;
+	}
+
+	my $translate = sub {
+	    my $rec = shift;
+
+	    if (my $path = $rec->get_header('Node-copyfrom-path')) {
+		$path = $prefix.$path;
+		$rec->set_header('Node-copyfrom-path' => $path );
+	    }
+	    if (my $rev = $rec->get_header('Node-copyfrom-rev')) {
+#		$rec->set_header('Node-copyfrom-rev' =>
+#		    scalar $self->find_local_rev( $rev, $self->source_uuid )  - 1);
+	    }
+	    
+	    if ($rec->get_header('Revision-number')) {
+		$| = 1;
+		$rev = $rec->get_header('Revision-number');
+		$prev = $rev if !$prev;
+		$rec->set_property('svm:headrev',$self->source_uuid.':'.$rev."\n");
+		printf STDERR "%s rev:%d\r",$progress->report( "%45b",$rev),$rev;
+	    }
+
+
+	    if ( my $path = $rec->get_header('Node-path') ) {
+		$path = $prefix.$path;
+		$rec->set_header('Node-path' => $path);
+	    }
+
+	};
+	$translate->( $record );
+	my $inc = $record->get_included_record;
+	$translate->( $inc ) if $inc;
+
+	if ($rev and $prev != $rev) {
+	    $self->_import_repos($header,$buf) if $prev > $self->fromrev;
+	    $buf = "";
+	    $prev = $rev;
+	}
+
+	$buf = $buf.$record->as_string;
+    }
+    # last one
+    if ($rev) {
+	$self->_import_repos($header, $buf)  if $prev > $self->fromrev;
+    }
+
+}
+
+sub _import_repos {
+    my $self = shift;
+    my ($header, $buf) = @_;
+    $buf = $header.$buf;
+    open my $fh, '<', \$buf;
+    my $feedback = '';
+    open my $fstream, '>', \$feedback;
+    my $ret = SVN::Repos::load_fs2( $self->repos, $fh, $fstream, $SVN::Repos::load_uuid_default, undef, 0, 0, undef, undef );
+    # (repos,dumpstream,feedback_stream,uuid_action,parent_dir,use_pre_commit_hook,use_post_commit_hook,cancel_func,cancel_baton,pool);
+    # XXX: display $feedback if we are in verbose / debug mode.
+    # and provide progress feedback in caller
+    return $ret;
+}
+
 =item relocate($newurl)
 
 =item with_lock($code)
@@ -369,10 +473,13 @@
 
     $logger->info(loc("Syncing %1", $self->url).($self->_backend->_relayed ? loc(" via %1", $self->server_url) : ""));
 
+    $self->{use_progress} = 1 unless SVK::Test->can('is_output');
+
     $self->mirror_changesets($torev,
         sub {
             my ( $changeset, $rev ) = @_;
-            $logger->info("Committed revision $rev from revision $changeset.");
+            $logger->info("Committed revision $rev from revision $changeset.")
+                unless $self->{use_progress};
         }, $fake_last
     );
     die $@ if $@;

Modified: trunk/lib/SVK/Mirror/Backend/SVNRa.pm
==============================================================================
--- trunk/lib/SVK/Mirror/Backend/SVNRa.pm	(original)
+++ trunk/lib/SVK/Mirror/Backend/SVNRa.pm	Fri Jul 18 19:01:58 2008
@@ -779,7 +779,12 @@
         }
         $ra = SVK::Mirror::Backend::SVNRaPipe->new( $ra, sub { shift @gen } );
     }
+    my $progress =
+      $self->mirror->{use_progress}
+      ? SVK::Notify->new->progress( max => scalar @$revs )
+      : undef;
     my $pool = SVN::Pool->new_default;
+    my $i = 0;
     for (@$revs) {
         $pool->clear;
         my ( $changeset, $metadata ) = @$_;
@@ -797,6 +802,8 @@
                 $ra->replay( $changeset, 0, 1, $editor );
                 $self->_after_replay($ra, $editor);
             }, $translate_from );
+        local $| = 1;
+        print STDERR $progress->report( "%45b %p\r", ++$i ) if $progress;
     }
     $self->_ra_finished($ra);
 }

Modified: trunk/lib/SVK/Notify.pm
==============================================================================
--- trunk/lib/SVK/Notify.pm	(original)
+++ trunk/lib/SVK/Notify.pm	Fri Jul 18 19:01:58 2008
@@ -209,5 +209,14 @@
     $self->flush ($path, 1);
 }
 
+sub progress {
+    my $self = shift;
+    require Time::Progress;
+    return if $self->{quiet};
+    my $progress = Time::Progress->new();
+    $progress->attr (@_);
+    return $progress;
+
+}
 
 1;

Added: trunk/t/mirror/sync-bootstrap.t
==============================================================================
--- (empty file)
+++ trunk/t/mirror/sync-bootstrap.t	Fri Jul 18 19:01:58 2008
@@ -0,0 +1,75 @@
+#!/usr/bin/perl -w
+use strict;
+use Test::More;
+use SVK::Test;
+eval { require SVN::Mirror; 1 } or plan skip_all => 'require SVN::Mirror';
+plan tests => 8;
+
+my ($xd, $svk) = build_test('test', 'm2', 'm3');
+
+our $output;
+
+my $tree = create_basic_tree($xd, '/test/');
+my $depot = $xd->find_depot('test');
+
+my $uri = uri($depot->repospath);
+
+my $dump = File::Temp->new;
+dump_all($depot => $dump);
+close $dump;
+
+is_output($svk, mirror => ['//m', $uri],
+          ["Mirror initialized.  Run svk sync //m to start mirroring."]);
+
+is_output($svk, mirror => ['/m2/m', $uri],
+          ["Mirror initialized.  Run svk sync /m2/m to start mirroring."]);
+$svk->sync('/m2/m');
+
+is_output($svk, mirror => ['--bootstrap='.$dump, '//m', $uri],
+	  ['Bootstrapping mirror from dump',
+	   'Mirror path \'//m\' synced from dumpfile.']);
+
+# compare normal mirror result and bootstrap mirror result
+my ($exp_mirror, $boot_mirror);
+open my $exp, '>', \$exp_mirror;
+open my $boot, '>', \$boot_mirror;
+dump_all($xd->find_depot('') => $boot);
+dump_all($xd->find_depot('m2') => $exp);
+$exp_mirror =~ s/UUID: .*//;
+$boot_mirror =~ s/UUID: .*//;
+# remove first svn-date (initial mirro)
+# 2007-08-09T14:43:18.137165Z
+$exp_mirror =~ s/\d{4}-\d{2}-\d{2}T[\d:.]+Z//;
+$boot_mirror =~ s/\d{4}-\d{2}-\d{2}T[\d:.]+Z//;
+
+is($boot_mirror, $exp_mirror, 'UUID should be the same'); # do something with UUID, they should be identical
+
+# now try with mirror, sync in single bootstrap command
+
+# try feed with incorrect file 
+is_output($svk, mirror => ['--bootstrap=./no-such-file', '/m3/m', $uri],
+          ["No such dump file: ./no-such-file."]);
+# this is real test
+is_output($svk, mirror => ['--bootstrap',$dump->filename, '/m3/m', $uri],
+	  ['Mirror initialized.  Run svk sync /m3/m to start mirroring.',
+	   'Bootstrapping mirror from dump',
+	   "Mirror path '/m3/m' synced from dumpfile."]);
+
+# compare UUID
+
+my ($boot_mirror2);
+open my $boot2, '>', \$boot_mirror2;
+dump_all($xd->find_depot('m3') => $boot2);
+$boot_mirror2 =~ s/UUID: .*//;
+# remove first svn-date (initial mirro)
+# 2007-08-09T14:43:18.137165Z
+$boot_mirror2 =~ s/\d{4}-\d{2}-\d{2}T[\d:.]+Z//;
+
+is($boot_mirror, $boot_mirror2, 'UUID should be the same');
+is($exp_mirror, $boot_mirror2, 'UUID should be the same');
+
+sub dump_all {
+    my ($depot, $output) = @_;
+    my $repos = $depot->repos;
+    SVN::Repos::dump_fs2($repos, $output, undef, 1, $repos->fs->youngest_rev, 0, 0, undef, undef);
+}

Added: trunk/utils/mipush
==============================================================================
--- (empty file)
+++ trunk/utils/mipush	Fri Jul 18 19:01:58 2008
@@ -0,0 +1,110 @@
+#!/usr/bin/perl -w
+use strict;
+use SVK;
+use SVK::XD;
+use SVN::Repos;
+use SVK::Util 'traverse_history';
+use Getopt::Long;
+#15:03 <clkao> mirror boot
+#15:03 <clkao> o mirror state initialisation, SVK::Mirror accessible
+#15:03 <clkao> o dump->read_record
+#15:03 <clkao> o translate record, add revprop for svm:head
+#15:03 <clkao> o svn::repos::load_from_dump($record->as_string)
+
+my $revspec;
+
+sub usage {
+        print <<EOUSAGE;
+Usage:  mipush repopath depotpath svn_dump_file
+    Reverse version of pullyu
+    
+    Example: ./mipush ./.svk/local //mirror/svk myproject-svn-dumpfile
+    
+EOUSAGE
+exit;
+}
+
+
+die unless GetOptions ("r|revision=s@" => \$revspec);
+
+use SVN::Dump 0.03;
+
+my $repospath = shift or usage();
+my $depotpath = shift or usage();
+my ($depotname, $path) = $depotpath =~ m|^/([^/]*)(/.*?)/?$|;
+my $file = shift or usage();
+
+my $repos = SVN::Repos::open($repospath) or die $!;
+my $depot = SVK::Depot->new({ depotname => $depotname, repos => $repos, repospath => $repospath});
+# TODO: for exists mirror, maybe we should remove it and create a new one?
+my $t  = SVK::Path->real_new({ depot => $depot, path => $path })
+    ->refresh_revision;
+my ( $m, $mpath ) = $t->is_mirrored;
+die $t->depotpath." is not a mirrored path.\n" if !$m;
+die $t->depotpath." is inside a mirrored path.\n" if $mpath;
+die "only whole repository mirrors are supported.\n" if length($mpath);
+
+
+my $prev = undef;
+my $rev = 0;
+my $dump = SVN::Dump->new( { file => $file } );
+my $prefix = $m->path.'/';
+my $buf;
+my $header;
+while ( my $record = $dump->next_record() ) {
+    if ($record->type eq 'format' || $record->type eq 'uuid') {
+	$header = $header.$record->as_string;
+	next;
+    }
+
+    my $translate = sub {
+	my $rec = shift;
+
+	if (my $path = $rec->get_header('Node-copyfrom-path')) {
+	    $path = $prefix.$path;
+	    $rec->set_header('Node-copyfrom-path' => $path );
+	}
+#	if (my $rev = $rec->get_header('Node-copyfrom-rev')) {
+#	    $rec->set_header('Node-copyfrom-rev' => 0);
+#		scalar $m->find_local_rev( $rev, $m->source_uuid ) - $rev);
+#	    warn "$rev changed to ". (scalar $m->find_local_rev( $rev, $m->source_uuid ) - $rev) ;
+#	}
+	
+	if ($rec->get_header('Revision-number')) {
+	    $rev = $rec->get_header('Revision-number');
+	    $prev = $rev if !$prev;
+	    $rec->set_property('svm:headrev',$m->source_uuid.':'.$rev."\n");
+	}
+
+
+	if ( my $path = $rec->get_header('Node-path') ) {
+	    $path = $prefix.$path;
+	    $rec->set_header('Node-path' => $path);
+	}
+
+    };
+    $translate->( $record );
+    my $inc = $record->get_included_record;
+    $translate->( $inc ) if $inc;
+
+    if ($rev and $prev != $rev) {
+	import_repos($header,$buf);
+	$buf = "";
+	$prev = $rev;
+    }
+
+    $buf = $buf.$record->as_string;
+}
+
+if ($rev) {
+    import_repos($header,$buf);
+}
+sub import_repos {
+    my ($header, $buf) = @_;
+    $buf = $header.$buf;
+    open my $fh, '<', \$buf;
+    my $ret = SVN::Repos::load_fs2( $repos, $fh, \*STDERR, $SVN::Repos::load_uuid_default, undef, 0, 0, undef, undef );
+    # (repos,dumpstream,feedback_stream,uuid_action,parent_dir,use_pre_commit_hook,use_post_commit_hook,cancel_func,cancel_baton,pool);
+    return $ret;
+}
+


More information about the svk-commit mailing list