[Bps-public-commit] RT-Extension-ExportImport branch, master, updated. abddc29a120fcf8d3e87cd5fce27ed47dae7d94b
Ruslan Zakirov
ruz at bestpractical.com
Wed Jan 19 09:43:05 EST 2011
The branch, master has been updated
via abddc29a120fcf8d3e87cd5fce27ed47dae7d94b (commit)
via 0f54869841531e9c5a98965d9e3b8eef51f702a7 (commit)
via 094612153a4a91b128810bb8a444a5311ae43ff0 (commit)
from 802f786c33e70085882a01fbc23e5c14c351c34b (commit)
Summary of changes:
lib/RT/App/Export.pm | 82 +----------------------------
lib/RT/App/Import.pm | 107 +++++++++++++++++++++++++++++++++++++-
lib/RT/Extension/ExportImport.pm | 33 +++++++++---
3 files changed, 132 insertions(+), 90 deletions(-)
- Log -----------------------------------------------------------------
commit 094612153a4a91b128810bb8a444a5311ae43ff0
Author: Ruslan Zakirov <ruz at bestpractical.com>
Date: Wed Jan 19 17:19:14 2011 +0300
new helper functions in generic class
diff --git a/lib/RT/Extension/ExportImport.pm b/lib/RT/Extension/ExportImport.pm
index a1988bd..0f66d54 100644
--- a/lib/RT/Extension/ExportImport.pm
+++ b/lib/RT/Extension/ExportImport.pm
@@ -39,12 +39,12 @@ sub export {
my $seen = $self->{'seen'}{ $args{'class'} } ||= {};
- my ($fh, $csv) = $self->csv( %args );
+ my ($fh, $csv) = $self->csv_writer( %args );
my $query = 'SELECT '. join(', ', map "$args{'alias'}.$_", @fields)
.' FROM '. $args{'query'};
- print "$query\n";
- print "\tbindings: ". join(', ', map "'$_'", @{ $args{'binds'} }) . "\n"
+ $self->debug($query);
+ $self->debug("\tbindings: ". join(', ', map "'$_'", @{ $args{'binds'} }) . "\n")
if @{ $args{'binds'} };
my $sth = $RT::Handle->SimpleQuery( $query, @{ $args{'binds'} } );
@@ -60,11 +60,11 @@ sub export {
$csv->print($fh, $row);
}
- print "Exported $counter rows\n";
+ $self->debug("Exported $counter rows\n");
return $counter;
}
-sub csv {
+sub csv_writer {
my $self = shift;
my %args = (
class => undef,
@@ -76,18 +76,26 @@ sub csv {
my $path = File::Spec->catfile( $self->output, $name . '.csv' );
my $exists = -e $path;
- open my $fh, '>>:raw', $path
+ open my $fh, '>>', $path
or die "Couldn't open '$path': $!";
- my $csv = Text::CSV->new ( { binary => 1 } );
-
+ my $csv = $self->csv;
unless ( $exists ) {
- print $fh '# RT '. $RT::VERSION .' '. $args{'class'} ."\n";
+ $csv->print( $fh, [ 'rt-export', $RT::VERSION, $args{'class'} ] );
+ use Data::Dumper;
+ print Dumper( [$self->fields( class => $args{'class'} ) ]);
$csv->print( $fh, [ $self->fields( class => $args{'class'} ) ] );
}
return ($fh, $csv);
}
+sub csv {
+ my $self = shift;
+ require Text::CSV_PP;
+ my $csv = Text::CSV_PP->new ( { binary => 1, eol => "\r\n" } );
+ return $csv;
+}
+
sub fields {
my $self = shift;
my %args = (@_);
@@ -101,6 +109,13 @@ sub table {
return $args{'class'}->new( $RT::SystemUser )->Table;
}
+sub debug {
+ my $self = shift;
+ return unless $self->{'debug'};
+
+ print STDOUT @_, "\n";
+}
+
=head1 AUTHOR
Ruslan Zakirov E<lt>ruz at bestpractical.comE<gt>
commit 0f54869841531e9c5a98965d9e3b8eef51f702a7
Author: Ruslan Zakirov <ruz at bestpractical.com>
Date: Wed Jan 19 17:20:19 2011 +0300
fixes in exporter
diff --git a/lib/RT/App/Export.pm b/lib/RT/App/Export.pm
index 6a8594e..6ca21d0 100644
--- a/lib/RT/App/Export.pm
+++ b/lib/RT/App/Export.pm
@@ -206,7 +206,7 @@ sub follow_cgm_self {
)
";
return $self->push_collection(
- class => 'RT::CachedGroupMembers',
+ class => 'RT::CachedGroupMember',
alias => 'cgm',
query => $query,
binds => $args{'binds'},
@@ -305,7 +305,7 @@ sub follow_transaction_values {
)
";
$self->push_collection(
- class => 'RT::Principals',
+ class => 'RT::Principal',
alias => 'p',
query => $query,
binds => [ @{ $args{'binds'} }, qw(AddWatcher DelWatcher) ],
@@ -322,7 +322,7 @@ sub follow_transaction_values {
)
";
$self->push_collection(
- class => 'RT::Links',
+ class => 'RT::Link',
alias => 'l',
query => $query,
binds => [ @{ $args{'binds'} }, qw(AddLink DeleteLink) ],
@@ -400,82 +400,6 @@ sub follow_links {
}
}
-sub export {
- my $self = shift;
- my %args = (
- class => undef,
- alias => undef,
- query => undef,
- binds => [],
- @_
- );
-
- my @fields = $self->fields( class => $args{'class'} );
-
- my $seen = $self->{'seen'}{ $args{'class'} } ||= {};
-
- my ($fh, $csv) = $self->csv( %args );
-
- my $query = 'SELECT '. join(', ', map "$args{'alias'}.$_", @fields)
- .' FROM '. $args{'query'};
- print "$query\n";
- print "\tbindings: ". join(', ', map "'$_'", @{ $args{'binds'} }) . "\n"
- if @{ $args{'binds'} };
-
- my $sth = $RT::Handle->SimpleQuery( $query, @{ $args{'binds'} } );
- unless ( $sth ) {
- die "Couldn't execute the query: ". $sth->error_message;
- }
-
- my $counter = 0;
- while ( my $row = $sth->fetchrow_arrayref ) {
- next if exists $seen->{ $row->[0] };
- $seen->{ $row->[0] } = undef;
- $counter++;
-
- $csv->print($fh, $row);
- }
- print "Exported $counter rows\n";
- return $counter;
-}
-
-sub csv {
- my $self = shift;
- my %args = (
- class => undef,
- @_
- );
-
- my $name = lc $args{'class'};
- $name =~ s/:/_/g;
- my $path = File::Spec->catfile( $self->output, $name . '.csv' );
- my $exists = -e $path;
-
- open my $fh, '>>:raw', $path
- or die "Couldn't open '$path': $!";
-
- my $csv = Text::CSV->new ( { binary => 1 } );
-
- unless ( $exists ) {
- print $fh '# RT '. $RT::VERSION .' '. $args{'class'} ."\n";
- $csv->print( $fh, [ $self->fields( class => $args{'class'} ) ] );
- }
- return ($fh, $csv);
-}
-
-sub fields {
- my $self = shift;
- my %args = (@_);
-
- return sort { $a eq 'id'? -1 : $a cmp $b } $RT::Handle->Fields( $self->table( %args ) );
-}
-
-sub table {
- my $self = shift;
- my %args = (@_);
- return $args{'class'}->new( $RT::SystemUser )->Table;
-}
-
=head1 AUTHOR
Ruslan Zakirov E<lt>ruz at bestpractical.comE<gt>
commit abddc29a120fcf8d3e87cd5fce27ed47dae7d94b
Author: Ruslan Zakirov <ruz at bestpractical.com>
Date: Wed Jan 19 17:21:25 2011 +0300
importer is now can import things
diff --git a/lib/RT/App/Import.pm b/lib/RT/App/Import.pm
index d3713aa..11b7a5a 100644
--- a/lib/RT/App/Import.pm
+++ b/lib/RT/App/Import.pm
@@ -25,6 +25,8 @@ sub init {
return $self;
}
+our @OPTIONS = ('input=s', 'to-version=s', 'debug!');
+
sub run {
my $proto = shift;
@@ -37,14 +39,115 @@ sub run {
Getopt::Long::GetOptionsFromArray( \@args, \%args, @OPTIONS );
my $self = $proto->new( %args );
- $proto->import;
+ $self->import_dir;
return 0;
}
sub input { return $_[0]->{'input'} }
-sub import {
+sub import_dir {
+ my $self = shift;
+
+ my $blob = File::Spec->catfile( $self->input, '*.csv' );
+ my @files = glob $blob;
+
+ foreach my $file ( @files ) {
+ $self->debug("About to process '$file'");
+
+ open my $fh, '<', $file
+ or die "Couldn't open '$file': $!";
+ my $csv = $self->csv;
+
+ my ($token, $version, $class) = @{ $csv->getline( $fh ) || [] };
+ unless ( $token eq 'rt-export' ) {
+ die "Doesn't look like an export from RT";
+ }
+
+ $self->debug("File contains $class records exported from RT $version");
+
+ my $fields = $csv->getline( $fh );
+ $self->debug("Fields ". join(', ', "'$_'", @$fields));
+
+ $self->import_class(
+ file => $file,
+ handle => $fh,
+ csv => $csv,
+ version => $version,
+ class => $class,
+ table => $self->table( class => $class ),
+ fields => $fields,
+ );
+
+ }
+}
+
+our %IMMUTABLE = map { $_ => 1 } qw(
+ RT::Transaction
+ RT::Attachment
+ RT::ObjectCustomFieldValue
+ RT::GroupMember
+ RT::CachedGroupMember
+);
+
+sub import_class {
+ my $self = shift;
+ my %args = @_;
+
+ my %current_field = map { lc($_) => 1 } $RT::Handle->Fields( $args{'table'} );
+
+ while ( my $row = $args{'csv'}->getline( $args{'handle'} ) ) {
+ my %row = map { lc($_) => shift(@$row) } @{ $args{'fields'} };
+
+ delete $row{$_} foreach grep !$current_field{ $_ }, keys %row;
+
+ $self->import_record( %args, row => \%row );
+ }
+
+}
+
+sub import_record {
+ my $self = shift;
+ my %args = @_;
+
+ my $query = 'SELECT id FROM '. $args{'table'} .' WHERE id = ?';
+
+ my $sth = $RT::Handle->SimpleQuery( $query, $args{'row'}{'id'} );
+ unless ( $sth ) {
+ die "Couldn't execute the query: ". $sth->error_message;
+ }
+
+ unless ( $sth->fetchrow_array ) {
+
+ $self->debug( "Inserting $args{'class'} #". $args{'row'}{'id'} );
+
+ my @fields = keys %{ $args{'row'} };
+ my $query =
+ 'INSERT INTO '. $args{'table'} .'('. join(', ', @fields ) .')'
+ .' VALUES ('. join(', ', ('?') x @fields ) .')';
+
+ my $sth = $RT::Handle->SimpleQuery( $query, map $args{'row'}{$_}, @fields );
+ unless ( $sth ) {
+ die "Couldn't execute the query: ". $sth->error_message;
+ }
+ } else {
+ if ( $IMMUTABLE{ $args{'class'} } ) {
+ $self->debug( "$args{'class'} #". $args{'row'}{'id'} .' exists and record is immutable. Skipping' );
+ return;
+ }
+
+ $self->debug( "Updating $args{'class'} #". $args{'row'}{'id'} );
+
+ my @fields = grep $_ ne 'id', keys %{ $args{'row'} };
+ my $query =
+ 'UPDATE '. $args{'table'} .' SET '. join(' AND ', map "$_ = ?", @fields ) .''
+ .' WHERE id = ?';
+
+ my $sth = $RT::Handle->SimpleQuery( $query, map $args{'row'}{$_}, @fields, $args{'row'}{'id'} );
+ unless ( $sth ) {
+ die "Couldn't execute the query: ". $sth->error_message;
+ }
+ }
}
1;
-----------------------------------------------------------------------
More information about the Bps-public-commit
mailing list