[Bps-public-commit] SD - A distributed issue tracker branch, master, updated. 8fd157776766b24cf96a8eca4fde9882ea00c892
jesse
jesse at bestpractical.com
Wed Jan 21 16:57:17 EST 2009
The branch, master has been updated
via 8fd157776766b24cf96a8eca4fde9882ea00c892 (commit)
from d401ed000a82562179634e85cc6a1ebec1c17f34 (commit)
Summary of changes:
lib/App/SD/Replica/trac.pm | 100 ++++++++
lib/App/SD/Replica/trac/PullEncoder.pm | 322 ++++++++++++++++++++++++
lib/App/SD/Replica/{rt => trac}/PushEncoder.pm | 18 +-
3 files changed, 431 insertions(+), 9 deletions(-)
create mode 100644 lib/App/SD/Replica/trac.pm
create mode 100644 lib/App/SD/Replica/trac/PullEncoder.pm
copy lib/App/SD/Replica/{rt => trac}/PushEncoder.pm (89%)
- Log -----------------------------------------------------------------
commit 8fd157776766b24cf96a8eca4fde9882ea00c892
Author: Jesse Vincent <jesse at bestpractical.com>
Date: Wed Jan 21 16:56:43 2009 -0500
Very first start at a stab for the Net::Trac support - Not expected to work yet
diff --git a/lib/App/SD/Replica/trac.pm b/lib/App/SD/Replica/trac.pm
new file mode 100644
index 0000000..c6b92bb
--- /dev/null
+++ b/lib/App/SD/Replica/trac.pm
@@ -0,0 +1,100 @@
+package App::SD::Replica::rt;
+use Moose;
+extends qw/App::SD::ForeignReplica/;
+
+use Params::Validate qw(:all);
+use Path::Class;
+use File::Temp 'tempdir';
+use Memoize;
+
+use constant scheme => 'trac';
+use constant pull_encoder => 'App::SD::Replica::trac::PullEncoder';
+use constant push_encoder => 'App::SD::Replica::trac::PushEncoder';
+
+
+use Prophet::ChangeSet;
+
+has trac => ( isa => 'Net::Trac::Connection', is => 'rw');
+has remote_url => ( isa => 'Str', is => 'rw');
+
+sub BUILD {
+ my $self = shift;
+
+ # Require rather than use to defer load
+ require Net::Trac;
+
+ my ( $server, $type, $query ) = $self->{url} =~ m/^trac:(.*?)\|(.*?)\|(.*)$/
+ or die
+ "Can't parse Trac server spec. Expected trac:http://example.com";
+ my $uri = URI->new($server);
+ my ( $username, $password );
+ if ( my $auth = $uri->userinfo ) {
+ ( $username, $password ) = split /:/, $auth, 2;
+ $uri->userinfo(undef);
+ }
+ $self->remote_url( $uri->as_string );
+ $self->rt_queue($type);
+ $self->rt_query( ( $query ? "($query) AND " : "" ) . " Queue = '$type'" );
+
+ ( $username, $password ) = $self->prompt_for_login( $uri, $username )
+ unless $password;
+ $self->trac(
+ Net::Trac::Connection->new(
+ url => 'http://trac.someproject.org',
+ user => $username,
+ password => $password
+ )
+ );
+
+}
+
+sub record_pushed_transactions {
+ my $self = shift;
+ my %args = validate( @_,
+ { ticket => 1, changeset => { isa => 'Prophet::ChangeSet' } } );
+
+ # walk through every transaction on the ticket, starting with the latest
+ for my $txn ( 'find all the transactions pushed upstream') {
+
+ # if the transaction id is older than the id of the last changeset
+ # we got from the original source of this changeset, we're done
+ last if $txn->id <= $self->last_changeset_from_source(
+ $args{changeset}->original_source_uuid
+ );
+
+ # if the transaction from RT is more recent than the most recent
+ # transaction we got from the original source of the changeset
+ # then we should record that we sent that transaction upstream
+ # XXX TODO - THIS IS WRONG - we should only be recording transactions we pushed
+ $self->record_pushed_transaction(
+ transaction => $txn->id,
+ changeset => $args{'changeset'},
+ record => $args{'ticket'}
+ );
+ }
+}
+
+
+=head2 uuid
+
+Return the replica's UUID
+
+=cut
+
+sub uuid {
+ my $self = shift;
+ return $self->uuid_for_url( $self->remote_url);
+}
+
+
+sub remote_uri_path_for_id {
+ my $self = shift;
+ my $id = shift;
+ return "/ticket/".$id;
+}
+
+
+__PACKAGE__->meta->make_immutable;
+no Moose;
+
+1;
diff --git a/lib/App/SD/Replica/trac/PullEncoder.pm b/lib/App/SD/Replica/trac/PullEncoder.pm
new file mode 100644
index 0000000..f90cd39
--- /dev/null
+++ b/lib/App/SD/Replica/trac/PullEncoder.pm
@@ -0,0 +1,322 @@
+package App::SD::Replica::trac::PullEncoder;
+use Moose;
+extends 'App::SD::ForeignReplica::PullEncoder';
+
+use Params::Validate qw(:all);
+use Memoize;
+use Time::Progress;
+
+has sync_source =>
+ ( isa => 'App::SD::Replica::trac',
+ is => 'rw');
+
+sub run {
+ my $self = shift;
+ my %args = validate(
+ @_,
+ { after => 1,
+ callback => 1,
+ }
+ );
+
+ my $first_rev = ( $args{'after'} + 1 ) || 1;
+
+ my $tickets = {};
+ my @transactions;
+
+ my @tickets = $self->find_matching_tickets();
+
+ $self->sync_source->log("No tickets found.") if @tickets == 0;
+
+ my $counter = 0;
+ $self->sync_source->log("Discovering ticket history");
+ my $progress = Time::Progress->new();
+ $progress->attr( max => $#tickets );
+ local $| = 1;
+ for my $id (@tickets) {
+ $counter++;
+ print $progress->report( "%30b %p Est: %E\r", $counter );
+
+ $self->sync_source->log( "Fetching ticket $id - $counter of " . scalar @tickets);
+ $tickets->{ $id } = $self->_translate_final_ticket_state(
+ $self->sync_source->trac->show( type => 'ticket', id => $id )
+ );
+ push @transactions, @{
+ $self->find_matching_transactions(
+ ticket => $id,
+ starting_transaction => $first_rev
+ )
+ };
+ }
+
+ my $txn_counter = 0;
+ my @changesets;
+ for my $txn ( sort { $b->{'id'} <=> $a->{'id'} } @transactions ) {
+ $txn_counter++;
+ $self->sync_source->log("Transcoding transaction @{[$txn->{'id'}]} - $txn_counter of ". scalar @transactions);
+ my $changeset = $self->transcode_one_txn( $txn, $tickets->{ $txn->{Ticket} } );
+ $changeset->created( $txn->{'Created'} );
+ next unless $changeset->has_changes;
+ unshift @changesets, $changeset;
+ }
+
+ my $cs_counter = 0;
+ for ( @changesets ) {
+ $self->sync_source->log("Applying changeset ".++$cs_counter . " of ".scalar @changesets);
+ $args{callback}->($_)
+ }
+}
+
+sub _translate_final_ticket_state {
+ my $self = shift;
+ my $ticket = shift;
+
+ # undefine empty fields, we'll delete after cleaning
+ $ticket->{$_} = undef for
+ grep defined $ticket->{$_} && $ticket->{$_} eq '',
+ keys %$ticket;
+
+ $ticket->{'id'} =~ s/^ticket\///g;
+
+ $ticket->{ $self->sync_source->uuid . '-' . lc($_) } = delete $ticket->{$_}
+ for qw(Queue id);
+
+ delete $ticket->{'Owner'} if lc($ticket->{'Owner'}) eq 'nobody';
+ $ticket->{'Owner'} = $self->resolve_user_id_to( email_address => $ticket->{'Owner'} )
+ if $ticket->{'Owner'};
+
+ # normalize names of watchers to variant with suffix 's'
+ foreach my $field (qw(Requestor Cc AdminCc)) {
+ if ( defined $ticket->{$field} && defined $ticket->{$field .'s'} ) {
+ die "It's impossible! Ticket has '$field' and '${field}s'";
+ } elsif ( defined $ticket->{$field} ) {
+ $ticket->{$field .'s'} = delete $ticket->{$field};
+ }
+ }
+
+ $ticket->{$_} = $self->unix_time_to_iso( $ticket->{$_} )
+ for grep defined $ticket->{$_}, qw(Created Resolved Told LastUpdated Due Starts Started);
+
+ $ticket->{$_} =~ s/ minutes$//
+ for grep defined $ticket->{$_}, qw(TimeWorked TimeLeft TimeEstimated);
+
+ $ticket->{'Status'} =~ $self->translate_status($ticket->{'Status'});
+
+ # delete undefined and empty fields
+ delete $ticket->{$_} for
+ grep !defined $ticket->{$_} || $ticket->{$_} eq '',
+ keys %$ticket;
+
+ return $ticket;
+}
+
+=head2 find_matching_tickets QUERY
+
+Returns a Trac::TicketSearch collection for all tickets found matching your QUERY hash.
+
+=cut
+
+sub find_matching_tickets {
+ my $self = shift;
+ my %query = (@_);
+ my $search = Net::Trac::TicketSearch->new( connection => $self->sync_source->trac );
+
+ $search->query(%query);
+
+ print $_->id, "\n" for @{ $search->results };
+
+ return $search->results;
+}
+
+=head2 find_matching_transactions { ticket => $id, starting_transaction => $num }
+
+Returns a reference to an array of all transactions (as hashes) on ticket $id after transaction $num.
+
+=cut
+
+sub find_matching_transactions {
+ my $self = shift;
+ my %args = validate( @_, { ticket => 1, starting_transaction => 1 } );
+ my @txns;
+
+ my $trac_handle = $self->sync_source->trac;
+
+ my @transactions = $rt_handle->get_transaction_ids( parent_id => $args{'ticket'} );
+ for my $txn ( sort @transactions) {
+ # Skip things we know we've already pulled
+ next if $txn < $args{'starting_transaction'};
+
+ # Skip things we've pushed
+ next if $self->sync_source->foreign_transaction_originated_locally($txn, $args{'ticket'});
+
+
+ my $txn_hash = $rt_handle->get_transaction(
+ parent_id => $args{'ticket'},
+ id => $txn,
+ type => 'ticket'
+ );
+ if ( my $attachments = delete $txn_hash->{'Attachments'} ) {
+ for my $attach ( split( /\n/, $attachments ) ) {
+ next unless ( $attach =~ /^(\d+):/ );
+ my $id = $1;
+ my $a = $rt_handle->get_attachment( parent_id => $args{'ticket'}, id => $id);
+
+ push( @{ $txn_hash->{_attachments} }, $a )
+ if ( $a->{Filename} );
+
+ }
+
+ }
+ push @txns, $txn_hash;
+ }
+ return \@txns;
+}
+
+sub transcode_one_txn {
+ my ( $self, $txn, $ticket ) = (@_);
+
+ my $sub = $self->can( '_recode_txn_' . $txn->{'Type'} );
+ unless ($sub) {
+ die "Transaction type $txn->{Type} (for transaction $txn->{id}) not implemented yet";
+ }
+
+ my $changeset = Prophet::ChangeSet->new(
+ { original_source_uuid => $self->sync_source->uuid,
+ original_sequence_no => $txn->{'id'},
+ creator => $self->resolve_user_id_to( email_address => $txn->{'Creator'} ),
+ }
+ );
+
+ if ( $txn->{'Ticket'} ne $ticket->{ $self->sync_source->uuid . '-id' }
+ && $txn->{'Type'} !~ /^(?:Comment|Correspond)$/ )
+ {
+ warn "Skipping a data change from a merged ticket"
+ . $txn->{'Ticket'} . ' vs '
+ . $ticket->{ $self->sync_source->uuid . '-id' };
+ next;
+ }
+
+ delete $txn->{'OldValue'} if ( $txn->{'OldValue'} eq '' );
+ delete $txn->{'NewValue'} if ( $txn->{'NewValue'} eq '' );
+
+ $sub->( $self, ticket => $ticket, txn => $txn, changeset => $changeset );
+ $self->translate_prop_names($changeset);
+
+ if ( my $attachments = delete $txn->{'_attachments'} ) {
+ for my $attach (@$attachments) {
+ $self->_recode_attachment_create(
+ ticket => $ticket,
+ txn => $txn,
+ changeset => $changeset,
+ attachment => $attach
+ );
+ }
+ }
+
+ return $changeset;
+}
+
+sub _recode_attachment_create {
+ my $self = shift;
+ my %args = validate( @_, { ticket => 1, txn => 1, changeset => 1, attachment => 1 } );
+ my $change = Prophet::Change->new(
+ { record_type => 'attachment',
+ record_uuid => $self->sync_source->uuid_for_url( $self->sync_source->remote_url . "/attachment/" . $args{'attachment'}->{'id'} ),
+ change_type => 'add_file'
+ }
+ );
+ $change->add_prop_change( name => 'content_type', old => undef, new => $args{'attachment'}->{'ContentType'});
+ $change->add_prop_change( name => 'created', old => undef, new => $args{'txn'}->{'Created'} );
+ $change->add_prop_change( name => 'creator', old => undef, new => $self->resolve_user_id_to( email_address => $args{'attachment'}->{'Creator'}));
+ $change->add_prop_change( name => 'content', old => undef, new => $args{'attachment'}->{'Content'});
+ $change->add_prop_change( name => 'name', old => undef, new => $args{'attachment'}->{'Filename'});
+ $change->add_prop_change( name => 'ticket', old => undef, new => $self->sync_source->uuid_for_remote_id( $args{'ticket'}->{ $self->sync_source->uuid . '-id'} ));
+ $args{'changeset'}->add_change( { change => $change } );
+}
+
+use HTTP::Date;
+
+sub unix_time_to_iso {
+ my $self = shift;
+ my $date = shift;
+
+ return undef if $date eq 'Not set';
+ return HTTP::Date::time2iso($date);
+}
+
+our %PROP_MAP = (
+ subject => 'summary',
+ status => 'status',
+ owner => 'owner',
+ initialpriority => '_delete',
+ finalpriority => '_delete',
+ told => '_delete',
+ requestor => 'reporter',
+ requestors => 'reporter',
+ cc => 'cc',
+ ccs => 'cc',
+ admincc => 'admin_cc',
+ adminccs => 'admin_cc',
+ refersto => 'refers_to',
+ referredtoby => 'referred_to_by',
+ dependson => 'depends_on',
+ dependedonby => 'depended_on_by',
+ hasmember => 'members',
+ memberof => 'member_of',
+ priority => 'priority_integer',
+ resolved => 'completed',
+ due => 'due',
+ creator => 'creator',
+ timeworked => 'time_worked',
+ timeleft => 'time_left',
+ timeestimated => 'time_estimated',
+ lastupdated => '_delete',
+ created => 'created',
+ queue => 'queue',
+ starts => '_delete',
+ started => '_delete',
+);
+
+sub translate_status {
+ my $self = shift;
+ my $status = shift;
+
+ $status =~ s/^resolved$/closed/;
+
+
+ return $status;
+}
+
+sub translate_prop_names {
+ my $self = shift;
+ my $changeset = shift;
+
+ for my $change ( $changeset->changes ) {
+ next unless $change->record_type eq 'ticket';
+
+ my @new_props;
+ for my $prop ( $change->prop_changes ) {
+ next if ( ( $PROP_MAP{ lc( $prop->name ) } || '' ) eq '_delete' );
+ $prop->name( $PROP_MAP{ lc( $prop->name ) } ) if $PROP_MAP{ lc( $prop->name ) };
+ # Normalize away undef -> "" and vice-versa
+ for (qw/new_value old_value/) {
+ $prop->$_("") if !defined ($prop->$_());
+ }
+ next if ( $prop->old_value eq $prop->new_value);
+
+ if ( $prop->name =~ /^cf-(.*)$/ ) {
+ $prop->name( 'custom-' . $1 );
+ }
+
+ push @new_props, $prop;
+
+ }
+ $change->prop_changes( \@new_props );
+
+ }
+ return $changeset;
+}
+
+__PACKAGE__->meta->make_immutable;
+no Moose;
+1;
diff --git a/lib/App/SD/Replica/trac/PushEncoder.pm b/lib/App/SD/Replica/trac/PushEncoder.pm
new file mode 100644
index 0000000..1609d96
--- /dev/null
+++ b/lib/App/SD/Replica/trac/PushEncoder.pm
@@ -0,0 +1,167 @@
+package App::SD::Replica::trac::PushEncoder;
+use Moose;
+use Params::Validate;
+use Path::Class;
+has sync_source =>
+ ( isa => 'App::SD::Replica::trac',
+ is => 'rw');
+
+
+sub integrate_change {
+ my $self = shift;
+ my ( $change, $changeset ) = validate_pos(
+ @_,
+ { isa => 'Prophet::Change' },
+ { isa => 'Prophet::ChangeSet' }
+ );
+ my $id;
+ local $@;
+ eval {
+ if ( $change->record_type eq 'ticket'
+ and $change->change_type eq 'add_file' )
+ {
+ $id = $self->integrate_ticket_create( $change, $changeset );
+ $self->sync_source->record_remote_id_for_pushed_record(
+ uuid => $change->record_uuid,
+ remote_id => $id
+ );
+
+ }
+ elsif (
+ $change->record_type eq 'attachment'
+ and $change->change_type eq 'add_file'
+
+ )
+ {
+ $id = $self->integrate_attachment( $change, $changeset );
+ }
+ elsif ( $change->record_type eq 'comment'
+ and $change->change_type eq 'add_file' )
+ {
+ $id = $self->integrate_comment( $change, $changeset );
+ }
+ elsif ( $change->record_type eq 'ticket' ) {
+ $id = $self->integrate_ticket_update( $change, $changeset );
+
+ }
+ else {
+ return undef;
+ }
+
+ $self->sync_source->record_pushed_transactions(
+ ticket => $id,
+ changeset => $changeset
+ );
+
+ };
+ warn $@ if $@;
+ return $id;
+}
+
+sub integrate_ticket_update {
+ my $self = shift;
+ my ( $change, $changeset ) = validate_pos(
+ @_,
+ { isa => 'Prophet::Change' },
+ { isa => 'Prophet::ChangeSet' }
+ );
+
+ # Figure out the remote site's ticket ID for this change's record
+ my $remote_ticket_id =
+ $self->sync_source->remote_id_for_uuid( $change->record_uuid );
+ my $ticket = Net::Trac::Ticket->new(
+ trac => $self->sync_source->trac,
+ id => $remote_ticket_id,
+ %{ $self->_recode_props_for_integrate($change) }
+ )->store();
+
+ return $remote_ticket_id;
+}
+
+sub integrate_ticket_create {
+ my $self = shift;
+ my ( $change, $changeset ) = validate_pos(
+ @_,
+ { isa => 'Prophet::Change' },
+ { isa => 'Prophet::ChangeSet' }
+ );
+
+ # Build up a ticket object out of all the record's attributes
+ my $ticket = Net::Trac::Ticket->new(
+ trac => $self->sync_source->trac,
+ queue => $self->sync_source->trac_queue(),
+ %{ $self->_recode_props_for_integrate($change) }
+ )->store( text => "Not yet pulling in ticket creation comment" );
+
+ return $ticket->id;
+}
+
+sub integrate_comment {
+ my $self = shift;
+ my ($change, $changeset) = validate_pos( @_, { isa => 'Prophet::Change' }, {isa => 'Prophet::ChangeSet'} );
+
+ # Figure out the remote site's ticket ID for this change's record
+
+ my %props = map { $_->name => $_->new_value } $change->prop_changes;
+
+ my $ticket_id = $self->sync_source->remote_id_for_uuid( $props{'ticket'} );
+ my $ticket = Net::Trac::Ticket->new( trac => $self->sync_source->trac, id => $ticket_id);
+
+ my %content = ( message => $props{'content'},
+ );
+
+ if ( ($props{'type'} ||'') eq 'comment' ) {
+ $ticket->comment( %content);
+ } else {
+ $ticket->correspond(%content);
+ }
+ return $ticket_id;
+}
+
+sub integrate_attachment {
+ my ($self, $change, $changeset ) = validate_pos( @_, { isa => 'App::SD::Replica::rt::PushEncoder'}, { isa => 'Prophet::Change' }, { isa => 'Prophet::ChangeSet' });
+
+
+ my %props = map { $_->name => $_->new_value } $change->prop_changes;
+ my $ticket_id = $self->sync_source->remote_id_for_uuid( $props{'ticket'});
+ my $ticket = Net::Trac::Ticket->new( trac => $self->sync_source->trac, id => $ticket_id );
+
+ my $tempdir = File::Temp::tempdir( CLEANUP => 1 );
+ my $file = file( $tempdir => ( $props{'name'} || 'unnamed' ) );
+ my $fh = $file->openw;
+ print $fh $props{content};
+ close $fh;
+ my %content = ( message => '(See attachments)', attachments => ["$file"]);
+ $ticket->correspond(%content);
+ return $ticket_id;
+}
+
+
+sub _recode_props_for_integrate {
+ my $self = shift;
+ my ($change) = validate_pos( @_, { isa => 'Prophet::Change' } );
+
+ my %props = map { $_->name => $_->new_value } $change->prop_changes;
+ my %attr;
+
+ for my $key ( keys %props ) {
+ next unless ( $key =~ /^(summary|queue|status|owner|custom)/ );
+ if ( $key =~ /^custom-(.*)/ ) {
+ $attr{cf}->{$1} = $props{$key};
+ } elsif ( $key eq 'summary' ) {
+ $attr{'subject'} = $props{summary};
+ } else {
+ $attr{$key} = $props{$key};
+ }
+ if ( $key eq 'status' ) {
+ $attr{$key} =~ s/^closed$/resolved/;
+ }
+ }
+ return \%attr;
+}
+
+
+__PACKAGE__->meta->make_immutable;
+no Moose;
+
+1;
-----------------------------------------------------------------------
More information about the Bps-public-commit
mailing list