[svk-commit] r2119 - in branches/mirror-pipeline: . lib/SVK/Mirror/Backend

clkao at bestpractical.com clkao at bestpractical.com
Mon Nov 6 04:14:02 EST 2006


Author: clkao
Date: Mon Nov  6 04:14:02 2006
New Revision: 2119

Modified:
   branches/mirror-pipeline/   (props changed)
   branches/mirror-pipeline/MANIFEST
   branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNRa.pm
   branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNRaPipe.pm
   branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNSync.pm
   branches/mirror-pipeline/lib/SVK/Path/View.pm

Log:
 r7224 at ubuntu:  clkao | 2006-11-06 04:41:59 +0000
 * Make SVN::Ra uses pipeline ra as well.
 * Disable pipeline in postcommit from view paths as there are unsafe txns.
 


Modified: branches/mirror-pipeline/MANIFEST
==============================================================================
--- branches/mirror-pipeline/MANIFEST	(original)
+++ branches/mirror-pipeline/MANIFEST	Mon Nov  6 04:14:02 2006
@@ -78,6 +78,7 @@
 lib/SVK/Editor/Merge.pm
 lib/SVK/Editor/Patch.pm
 lib/SVK/Editor/Rename.pm
+lib/SVK/Editor/Serialize.pm
 lib/SVK/Editor/Sign.pm
 lib/SVK/Editor/Status.pm
 lib/SVK/Editor/SubTree.pm
@@ -117,6 +118,7 @@
 lib/SVK/MimeDetect/Internal.pm
 lib/SVK/Mirror.pm
 lib/SVK/Mirror/Backend/SVNRa.pm
+lib/SVK/Mirror/Backend/SVNRaPipe.pm
 lib/SVK/Mirror/Backend/SVNSync.pm
 lib/SVK/MirrorCatalog.pm
 lib/SVK/Notify.pm

Modified: branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNRa.pm
==============================================================================
--- branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNRa.pm	(original)
+++ branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNRa.pm	Mon Nov  6 04:14:02 2006
@@ -7,6 +7,8 @@
 use SVN::Client ();
 use SVK::I18N;
 use SVK::Editor;
+use SVK::Mirror::Backend::SVNRaPipe;
+
 use Class::Autouse qw(SVK::Editor::SubTree SVK::Editor::CopyHandler);
 
 use constant OK => $SVN::_Core::SVN_NO_ERROR;
@@ -22,7 +24,7 @@
 
 # for this: things without _'s will probably move to base
 # SVK::Mirror::Backend
-__PACKAGE__->mk_accessors(qw(mirror _config _auth_baton _auth_ref _auth_baton source_root source_path fromrev _has_replay _cached_ra));
+__PACKAGE__->mk_accessors(qw(mirror _config _auth_baton _auth_ref _auth_baton source_root source_path fromrev _has_replay _cached_ra use_pipeline));
 
 =head1 NAME
 
@@ -54,7 +56,7 @@
 
 sub load {
     my ($class, $mirror) = @_;
-    my $self = $class->SUPER::new( { mirror => $mirror } );
+    my $self = $class->SUPER::new( { mirror => $mirror, use_pipeline => 1 } );
     my $t = $mirror->get_svkpath;
     die loc( "%1 is not a mirrored path.\n", $t->depotpath )
         unless $t->root->check_path( $mirror->path );
@@ -85,7 +87,7 @@
 sub create {
     my ($class, $mirror, $backend, $args, $txn, $editor) = @_;
 
-    my $self = $class->SUPER::new({ mirror => $mirror });
+    my $self = $class->SUPER::new({ mirror => $mirror, use_pipeline => 1 });
 
     my $ra = $self->_new_ra;
 
@@ -255,6 +257,7 @@
 sub _ra_finished {
     my ($self, $ra) = @_;
     return if $self->_cached_ra;
+    return if ref($ra) eq 'SVK::Mirror::Backend::SVNRaPipe';
     $self->_cached_ra( $ra );
 }
 
@@ -457,7 +460,7 @@
 }
 
 sub sync_changeset {
-    my ($self, $changeset, $metadata, $callback) = @_;
+    my ( $self, $changeset, $metadata, $ra, $extra_prop, $callback ) = @_;
     my $t = $self->mirror->get_svkpath;
     my ( $editor, undef, %opt ) = $t->get_editor(
         ignore_mirror => 1,
@@ -470,18 +473,41 @@
             $callback->( $changeset, $_[0] ) if $callback;
         }
     );
-    # XXX: sync relayed revmap as well
-    $opt{txn}->change_prop('svm:headrev', $self->mirror->server_uuid.":$changeset\n");
 
-    my $ra = $self->_new_ra;
-    if ( my $revprop = $self->mirror->depot->mirror->revprop ) {
-        my $prop = $ra->rev_proplist($changeset);
-        for (@$revprop) {
-            $opt{txn}->change_prop( $_, $prop->{$_} )
-                if exists $prop->{$_};
+    for (keys %$extra_prop) {
+	$opt{txn}->change_prop( $_, $extra_prop->{$_} );
+    }
+    $self->_revmap_prop( $opt{txn}, $changeset );
+
+    $editor = $self->_get_sync_editor($editor, $t);
+    $ra->replay( $changeset, 0, 1, $editor );
+    $self->_after_replay($ra, $editor);
+
+    return;
+
+}
+
+sub _after_replay {
+    my ($self, $ra, $editor) = @_;
+    if ( $editor->isa('SVK::Editor::SubTree') ) {
+	my $baton = $editor->anchor_baton;
+        if ( $editor->needs_touch ) {
+            $editor->change_dir_prop( $baton, 'svk:mirror' => undef );
         }
+	if (!$editor->changes) {
+	    $editor->abort_edit;
+	    return;
+	}
+        $editor->close_directory($baton);
     }
 
+    $editor->close_edit;
+    return;
+
+}
+
+sub _get_sync_editor {
+    my ($self, $editor, $target) = @_;
     $editor = SVK::Editor::CopyHandler->new(
         _editor => $editor,
         cb_copy => sub {
@@ -489,7 +515,7 @@
             return ( $path, $rev ) if $rev == -1;
             my $source_path = $self->source_path;
             $path =~ s/^\Q$self->{source_path}//;
-            return $t->as_url(
+            return $target->as_url(
                 1,
                 $self->mirror->path . $path,
                 $self->find_rev_from_changeset($rev)
@@ -500,7 +526,6 @@
     # ra->replay gives us editor calls based on repos root not
     # base uri, so we need to get the correct subtree.
     my $baton;
-    my $pool = SVN::Pool->new_default;
     if ( length $self->source_path ) {
         my $anchor = substr( $self->source_path, 1 );
         $baton  = $editor->open_root(-1);      # XXX: should use $t->revision
@@ -511,34 +536,55 @@
             }
         );
     }
-    $ra->replay( $changeset, 0, 1, $editor );
-    $self->_ra_finished($ra);
-    if ( length $self->source_path ) {
-        $editor->close_directory($baton);
-        if ( $editor->needs_touch ) {
-            $editor->change_dir_prop( $baton, 'svk:mirror' => undef );
-        }
-    }
-    if ( $editor->isa('SVK::Editor::SubTree') && !$editor->changes ) {
-        $editor->abort_edit;
-    } else {
-        $editor->close_edit;
-    }
-    return;
+    return $editor;
+}
 
+sub _revmap_prop {
+    my ($self, $txn, $changeset) = @_;
+    $txn->change_prop('svm:headrev', $self->mirror->server_uuid.":$changeset\n");
 }
 
+
 =item mirror_changesets
 
 =cut
 
 sub mirror_changesets {
     my ( $self, $torev, $callback ) = @_;
-
     $self->mirror->with_lock( 'mirror',
         sub {
-            $self->traverse_new_changesets(
-                sub { $self->sync_changeset( @_, $callback ) }, $torev );
+	    $self->refresh;
+	    my @revs;
+            $self->traverse_new_changesets( sub { push @revs, [@_] }, $torev );
+	    # prepare generator for pipelined ra
+	    my @gen;
+	    my $revprop = $self->mirror->depot->mirror->revprop; # XXX: this is so wrong
+	    return unless @revs;
+	    my $ra = $self->_new_ra;
+	    if ($self->use_pipeline) {
+		for (@revs) {
+		    push @gen, ['rev_proplist', $_->[0]] if $revprop;
+		    push @gen, ['replay', $_->[0], 0, 1, 'EDITOR'];
+		}
+		$ra = SVK::Mirror::Backend::SVNRaPipe->new($ra, sub { shift @gen });
+	    }
+	    my $pool = SVN::Pool->new_default;
+	    for (@revs) {
+		$pool->clear;
+		my ($changeset, $metadata) = @$_;
+		my $extra_prop = {};
+		if ( $revprop ) {
+		    my $prop = $ra->rev_proplist($changeset);
+		    for (@$revprop) {
+			$extra_prop->{$_}= $prop->{$_}
+			    if exists $prop->{$_};
+		    }
+		}
+		$self->sync_changeset( $changeset, $metadata, $ra,
+				       $extra_prop,
+				       $callback );
+	    }
+	    $self->_ra_finished($ra);
         }
     );
 }

Modified: branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNRaPipe.pm
==============================================================================
--- branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNRaPipe.pm	(original)
+++ branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNRaPipe.pm	Mon Nov  6 04:14:02 2006
@@ -2,7 +2,7 @@
 use strict;
 
 use base 'Class::Accessor::Fast';
-__PACKAGE__->mk_accessors(qw(ra requests fh unsent_buf buf_call current_editors));
+__PACKAGE__->mk_accessors(qw(ra requests fh unsent_buf buf_call current_editors pid));
 
 use POSIX 'EPIPE';
 use Socket;
@@ -98,6 +98,7 @@
 
     if (my $pid = fork) {
 	close $p;
+	$self->pid($pid);
 	return $self;
     }
     else {
@@ -106,6 +107,7 @@
     }
 
     $self->fh($p);
+    $File::Temp::KEEP_ALL = 1;
     # Begin external process for buffered ra requests.
     my $max_editor_in_buf = 5;
     my $pool = SVN::Pool->new_default;
@@ -161,7 +163,7 @@
 	    die unless UNIVERSAL::isa($arg, 'SVK::Editor');
 	    return $arg;
 	}
-	die if ($_ cmp $arg);
+	Carp::confess "pipeline ra error: got $arg but expecting $_" if ($_ cmp $arg);
     }
     die join(',', at arg) if @arg;
 }
@@ -221,9 +223,16 @@
 	}
     }
     else {
+	return if $func eq 'close_edit';
 	$ret = $editor->$func(@arg, $pool);
     }
     return $ret;
 }
 
+sub DESTROY {
+    my $self = shift;
+    return unless $self->pid;
+    wait;
+}
+
 1;

Modified: branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNSync.pm
==============================================================================
--- branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNSync.pm	(original)
+++ branches/mirror-pipeline/lib/SVK/Mirror/Backend/SVNSync.pm	Mon Nov  6 04:14:02 2006
@@ -49,113 +49,26 @@
     $self->mirror->depot->reposfs->change_rev_prop( 0, 'svn:svnsync:from-url',  $self->mirror->url );
 }
 
-sub find_rev_from_changeset {
-    return $_[0];
-}
-
-sub sync_changeset {
-    my ( $self, $changeset, $metadata, $callback ) = @_;
-    my $t = $self->mirror->get_svkpath('/');
-    my ( $editor, undef, %opt ) = $t->get_editor(
-        ignore_mirror => 1,
-        message       => $metadata->{message},
-        author        => $metadata->{author},
-        callback      => sub {
-            $t->repos->fs->change_rev_prop( $_[0], 'svn:date',
-                $metadata->{date} );
-            $self->fromrev( $_[0] );
-            $callback->( $changeset, $_[0] ) if $callback;
-        }
-    );
+sub find_rev_from_changeset { $_[0] }
 
-    my $ra = $self->_new_ra;
-    my $pool = SVN::Pool->new_default;
-    if ( my $revprop = $self->mirror->depot->mirror->revprop ) {
-        my $prop = $ra->rev_proplist($changeset);
-        for (@$revprop) {
-            $opt{txn}->change_prop( $_, $prop->{$_} )
-                if exists $prop->{$_};
-        }
-    }
+sub _revmap_prop { }
 
-    $editor = SVK::Editor::CopyHandler->new(
+sub _get_sync_editor {
+    my ($self, $editor, $target) = @_;
+    return SVK::Editor::CopyHandler->new(
         _editor => $editor,
         cb_copy => sub {
             my ( $editor, $path, $rev ) = @_;
             return ( $path, $rev ) if $rev == -1;
             $path =~ s{^\Q/}{};
-            return $t->as_url( 1, $path, $rev );
-        }
-    );
-
-    $ra->replay( $changeset, 0, 1, $editor );
-    $self->_ra_finished($ra);
-    $editor->close_edit;
-    return;
-
-}
-
-=item mirror_changesets
-
-=cut
-
-sub mirror_changesets {
-    my ( $self, $torev, $callback ) = @_;
-    $self->mirror->with_lock( 'mirror',
-        sub {
-	    $self->refresh;
-	    my @revs;
-            $self->traverse_new_changesets( sub { push @revs, [@_] }, $torev );
-	    # prepare generator for pipelined ra
-	    my @gen;
-	    for ($revs[0][0]..$revs[-1][0]) {
-		push @gen, ['rev_proplist', $_], ['replay', $_, 0, 1, 'EDITOR'];
-	    }
-	    require SVK::Mirror::Backend::SVNRaPipe;
-	    my $pra = SVK::Mirror::Backend::SVNRaPipe->new($self->_new_ra, sub { shift @gen });
-	    for (@revs) {
-		$self->evil_sync_changeset( @$_, $pra, $callback );
-	    }
+            return $target->as_url( 1, $path, $rev );
         }
-    );
+    )
 }
 
-sub evil_sync_changeset {
-    my ( $self, $changeset, $metadata, $ra, $callback ) = @_;
-    my $t = $self->mirror->get_svkpath('/');
-    my ( $editor, undef, %opt ) = $t->get_editor(
-        ignore_mirror => 1,
-        message       => $metadata->{message},
-        author        => $metadata->{author},
-        callback      => sub {
-            $t->repos->fs->change_rev_prop( $_[0], 'svn:date',
-                $metadata->{date} );
-            $self->fromrev( $_[0] );
-            $callback->( $changeset, $_[0] ) if $callback;
-        }
-    );
-
-    my $pool = SVN::Pool->new_default;
-    if ( my $revprop = $self->mirror->depot->mirror->revprop ) {
-        my $prop = $ra->rev_proplist($changeset);
-        for (@$revprop) {
-            $opt{txn}->change_prop( $_, $prop->{$_} )
-                if exists $prop->{$_};
-        }
-    }
-
-    $editor = SVK::Editor::CopyHandler->new(
-        _editor => $editor,
-        cb_copy => sub {
-            my ( $editor, $path, $rev ) = @_;
-            return ( $path, $rev ) if $rev == -1;
-            $path =~ s{^\Q/}{};
-            return $t->as_url( 1, $path, $rev );
-        }
-    );
-    $ra->replay( $changeset, 0, 1, $editor );
-    return;
-
+sub _after_replay {
+    my ($self, $ra, $editor) = @_;
+    $editor->close_edit;
 }
 
 sub _relayed { }

Modified: branches/mirror-pipeline/lib/SVK/Path/View.pm
==============================================================================
--- branches/mirror-pipeline/lib/SVK/Path/View.pm	(original)
+++ branches/mirror-pipeline/lib/SVK/Path/View.pm	Mon Nov  6 04:14:02 2006
@@ -55,7 +55,9 @@
     }
 
     my ($editor, $inspector, %extra) = $self->source->new(path => $actual_anchor)->get_editor(%arg);
-
+    # XXX: view has txns, not very happy with forked processes.
+    $extra{mirror}->_backend->use_pipeline(0)
+        if $extra{mirror} && $extra{mirror}->_backend->isa('SVK::Mirror::Backend::SVNRa');
     my $prefix = abs2rel($self->source->path_anchor,
 			 $actual_anchor => undef, '/');
 


More information about the svk-commit mailing list