[Rt-commit] rt branch, 4.2/serializer-importer, created. rt-4.0.2-272-g0f555d0

Alex Vandiver alexmv at bestpractical.com
Fri Jan 27 18:16:42 EST 2012


The branch, 4.2/serializer-importer has been created
        at  0f555d02e14b8f1d2a6d241e27d1920d51bd4c53 (commit)

- Log -----------------------------------------------------------------
commit 4885990e2ed3f0a51f1adb8a75bfee7ca7ec06a9
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Sun Nov 6 22:24:30 2011 -0500

    Dependency walker

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
new file mode 100644
index 0000000..5799fca
--- /dev/null
+++ b/lib/RT/DependencyWalker.pm
@@ -0,0 +1,201 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2011 Best Practical Solutions, LLC
+#                                          <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions,  LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+package RT::DependencyWalker;
+
+use strict;
+use warnings;
+
+use RT::DependencyWalker::Dependencies;
+
+sub new {
+    my $class = shift;
+    my $self = bless {}, $class;
+    $self->Init(@_);
+    return $self;
+}
+
+sub Init {
+    my $self = shift;
+    my %args = (
+        First         => "top",
+        @_
+    );
+
+    $self->{first}    = $args{First};
+    $self->{stack}    = [];
+}
+
+sub PushObj {
+    my $self = shift;
+    push @{$self->{stack}}, { object => $_ }
+        for @_;
+}
+
+sub Walk {
+    my $self = shift;
+
+    $self->PushObj( @_ );
+
+    $self->{visited} = {};
+    $self->{seen}    = {};
+
+    my $stack = $self->{stack};
+    while (@{$stack}) {
+        my %frame = %{ shift @{$stack} };
+        $self->{top}     = [];
+        $self->{replace} = [];
+        $self->{bottom}  = [];
+        my $ref = $frame{object};
+        if ($ref->isa("RT::Record")) {
+            $self->Process(%frame);
+        } else {
+            $ref->FindAllRows;
+            while (my $obj = $ref->Next) {
+                $self->Process(%frame, object => $obj );
+            }
+        }
+        unshift @{$stack}, @{$self->{replace}};
+        unshift @{$stack}, @{$self->{top}};
+        push    @{$stack}, @{$self->{bottom}};
+    }
+}
+
+sub Process {
+    my $self = shift;
+    my %args = (
+        object    => undef,
+        direction => undef,
+        from      => undef,
+        @_
+    );
+
+    my $obj = $args{object};
+    return if $obj->isa("RT::System");
+
+    my $uid = $obj->UID;
+    if (exists $self->{visited}{$uid}) {
+        # Already visited -- no-op
+        $self->Again(%args);
+    } elsif (exists $obj->{satisfied}) {
+        # All dependencies visited -- time to visit
+        $self->Visit(%args);
+        $self->{visited}{$uid}++;
+    } elsif (exists $self->{seen}{$uid}) {
+        # All of the dependencies are on the stack already.  We may not
+        # have gotten to them, but we will eventually.  This _may_ be a
+        # cycle, but true cycle detection is too memory-intensive, as it
+        # requires keeping track of the history of how each dep got
+        # added to the stack, all of the way back.
+        $self->ForcedVisit(%args);
+        $self->{visited}{$uid}++;
+    } else {
+        # Nothing known about this previously; add its deps to the
+        # stack, then objects it refers to.
+        return if defined $args{from}
+            and not $self->Observe(%args);
+        my $deps = RT::DependencyWalker::Dependencies->new;
+        $obj->Dependencies($self, $deps);
+        # Shove it back for later
+        push @{$self->{replace}}, \%args;
+        if ($self->{first} eq "top") {
+            # Top-first; that is, visit things we point to first,
+            # then deal with us, then deal with things that point to
+            # us.  For serialization.
+            $self->PrependDeps( out => $deps, $uid );
+            $self->AppendDeps(  in  => $deps, $uid );
+        } else {
+            # Bottom-first; that is, deal with things that point to
+            # us first, then deal with us, then deal with things we
+            # point to.  For removal.
+            $self->PrependDeps( in => $deps, $uid );
+            $self->AppendDeps( out => $deps, $uid );
+        }
+        $obj->{satisfied}++;
+        $self->{seen}{$uid}++;
+    }
+}
+
+sub Observe { 1 }
+
+sub Again {}
+
+sub Visit {}
+
+sub ForcedVisit {
+    my $self = shift;
+    $self->Visit( @_ );
+}
+
+sub AppendDeps {
+    my $self = shift;
+    my ($dir, $deps, $from) = @_;
+    for my $obj (@{$deps->{$dir}}) {
+        next if $obj->isa("RT::Record") and not $obj->id;
+        push @{$self->{bottom}}, {
+            object    => $obj,
+            direction => $dir,
+            from      => $from,
+        };
+    }
+}
+
+sub PrependDeps {
+    my $self = shift;
+    my ($dir, $deps, $from) = @_;
+    for my $obj (@{$deps->{$dir}}) {
+        next if $obj->isa("RT::Record") and not $obj->id;
+        unshift @{$self->{top}}, {
+            object    => $obj,
+            direction => $dir,
+            from      => $from,
+        };
+    }
+}
+
+1;
diff --git a/lib/RT/DependencyWalker/Dependencies.pm b/lib/RT/DependencyWalker/Dependencies.pm
new file mode 100644
index 0000000..b17e4ab
--- /dev/null
+++ b/lib/RT/DependencyWalker/Dependencies.pm
@@ -0,0 +1,65 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2011 Best Practical Solutions, LLC
+#                                          <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions,  LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+package RT::DependencyWalker::Dependencies;
+
+use strict;
+use warnings;
+
+sub new {
+    my $class = shift;
+    return bless {out => [], in => []}, $class;
+}
+
+sub Add {
+    my $self = shift;
+    my ($dir, $obj) = @_;
+    push @{$self->{$dir}}, $obj;
+}
+
+1;

commit 02cf361b87e472f8e3597b52ff5c1fd739cb1872
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Sun Nov 6 22:28:18 2011 -0500

    Walk large collections in batches of 100 results
    
    Instead of loading all of a large collection into memory at once, load
    only batches of 100 results at a time, pushing the remaining collection
    back onto the stack each time, to revisit later.

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index 5799fca..9df05c0 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -95,10 +95,21 @@ sub Walk {
         if ($ref->isa("RT::Record")) {
             $self->Process(%frame);
         } else {
-            $ref->FindAllRows;
+            unless ($ref->{unrolled}) {
+                $ref->FindAllRows;
+                $ref->RowsPerPage( 100 );
+                $ref->FirstPage;
+                $ref->{unrolled}++;
+            }
+            my $last;
             while (my $obj = $ref->Next) {
+                $last = $obj->Id;
                 $self->Process(%frame, object => $obj );
             }
+            if (defined $last) {
+                $ref->NextPage;
+                push @{$self->{replace}}, \%frame;
+            }
         }
         unshift @{$stack}, @{$self->{replace}};
         unshift @{$stack}, @{$self->{top}};

commit 50fd9d2c3565e2ba663dfc565576c4c40e3373eb
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Sun Nov 6 22:29:51 2011 -0500

    Add the concept of a UID to records

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index fcc7bed..b17f518 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -1937,6 +1937,12 @@ sub WikiBase {
     return RT->Config->Get('WebPath'). "/index.html?q=";
 }
 
+sub UID {
+    my $self = shift;
+    warn Carp::longmess("No ID for $self") unless defined $self->Id;
+    return "@{[ref $self]}-$RT::Organization-@{[$self->Id]}";
+}
+
 RT::Base->_ImportOverlays();
 
 1;

commit a572ce2d0f9718d243e68c353417b61e5366f678
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Sun Nov 6 22:33:14 2011 -0500

    Base-bones Dependencies method, which examines _Accessible for two common fields

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index b17f518..1ebc210 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -1943,6 +1943,19 @@ sub UID {
     return "@{[ref $self]}-$RT::Organization-@{[$self->Id]}";
 }
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+    for my $col (qw/Creator LastUpdatedBy/) {
+        if ( $self->_Accessible( $col, 'read' ) ) {
+            next unless $self->$col;
+            my $obj = RT::Principal->new( $self->CurrentUser );
+            $obj->Load( $self->$col );
+            $deps->Add( out => $obj->Object );
+        }
+    }
+}
+
 RT::Base->_ImportOverlays();
 
 1;

commit b1219a80db251fcffae63c2b3484b673c84668db
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Sun Nov 6 22:35:36 2011 -0500

    Attributes on objects

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 1ebc210..5dcac79 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -1954,6 +1954,10 @@ sub Dependencies {
             $deps->Add( out => $obj->Object );
         }
     }
+
+    # Object attributes, we have to check on every object
+    my $objs = $self->Attributes;
+    $deps->Add( in => $objs );
 }
 
 RT::Base->_ImportOverlays();

commit 9cd44adf155292be83421c0810eb284eb72b3eef
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Sun Nov 6 22:36:09 2011 -0500

    Transactions, optimized to object types that have them

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 5dcac79..fff6997 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -1958,6 +1958,19 @@ sub Dependencies {
     # Object attributes, we have to check on every object
     my $objs = $self->Attributes;
     $deps->Add( in => $objs );
+
+    # Transactions
+    if (   $self->isa("RT::Ticket")
+        or $self->isa("RT::User")
+        or $self->isa("RT::Group")
+        or $self->isa("RT::Article")
+        or $self->isa("RT::Queue") )
+    {
+        $objs = RT::Transactions->new( $self->CurrentUser );
+        $objs->Limit( FIELD => 'ObjectType', VALUE => ref $self );
+        $objs->Limit( FIELD => 'ObjectId', VALUE => $self->id );
+        $deps->Add( in => $objs );
+    }
 }
 
 RT::Base->_ImportOverlays();

commit bca35585f27b4ec4f679ea983a5e5db7ea52bbf9
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Sun Nov 6 22:36:32 2011 -0500

    ObjectCustomFieldValues, optimized to object types that have them

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index fff6997..494eab6 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -1971,6 +1971,18 @@ sub Dependencies {
         $objs->Limit( FIELD => 'ObjectId', VALUE => $self->id );
         $deps->Add( in => $objs );
     }
+
+    # Object custom field values
+    if (   $self->isa("RT::Transaction")
+        or $self->isa("RT::Ticket")
+        or $self->isa("RT::User")
+        or $self->isa("RT::Group")
+        or $self->isa("RT::Queue")
+        or $self->isa("RT::Article") )
+    {
+        $objs = $self->CustomFieldValues; # Actually OCFVs
+        $deps->Add( in => $objs );
+    }
 }
 
 RT::Base->_ImportOverlays();

commit f5dd83f8ba12d18c6ff9df7887912727a267e33b
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Sun Nov 6 23:47:00 2011 -0500

    Ensure that RT::Link's ClassAccessible is built early enough

diff --git a/lib/RT.pm b/lib/RT.pm
index 8d70af1..f1e27cc 100644
--- a/lib/RT.pm
+++ b/lib/RT.pm
@@ -423,6 +423,8 @@ sub InitClasses {
     require RT::ObjectTopics;
     require RT::Topic;
     require RT::Topics;
+    require RT::Link;
+    require RT::Links;
 
     # on a cold server (just after restart) people could have an object
     # in the session, as we deserialize it so we never call constructor
@@ -446,6 +448,7 @@ sub InitClasses {
         RT::ObjectCustomField
         RT::ObjectCustomFieldValue
         RT::Attribute
+        RT::Link
     );
 
     if ( $args{'Heavy'} ) {

commit d067fb0fe85b06cf1dfad986dd9f59aeeda67a12
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Sun Nov 6 23:59:23 2011 -0500

    Dependencies

diff --git a/lib/RT/ACE.pm b/lib/RT/ACE.pm
index 90220cd..974150b 100644
--- a/lib/RT/ACE.pm
+++ b/lib/RT/ACE.pm
@@ -744,6 +744,16 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->PrincipalObj->Object );
+    $deps->Add( out => $self->Object );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Article.pm b/lib/RT/Article.pm
index fe3e391..ca63c1d 100644
--- a/lib/RT/Article.pm
+++ b/lib/RT/Article.pm
@@ -862,6 +862,15 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+    $deps->Add( out => $self->ClassObj );
+    $deps->Add( in => $self->Topics );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index 0f6fcd2..c47efbd 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -1035,6 +1035,14 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+    $deps->Add( out => $self->TransactionObj );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Attribute.pm b/lib/RT/Attribute.pm
index e6eec4f..101cc8f 100644
--- a/lib/RT/Attribute.pm
+++ b/lib/RT/Attribute.pm
@@ -627,6 +627,14 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+    $deps->Add( out => $self->Object );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Class.pm b/lib/RT/Class.pm
index c7c8a63..29dc37a 100644
--- a/lib/RT/Class.pm
+++ b/lib/RT/Class.pm
@@ -616,6 +616,48 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    my $articles = RT::Articles->new( $self->CurrentUser );
+    $articles->Limit( FIELD => "Class", VALUE => $self->Id );
+    $deps->Add( in => $articles );
+
+    my $topics = RT::Topics->new( $self->CurrentUser );
+    $topics->LimitToObject( $self );
+    $deps->Add( in => $topics );
+
+    my $objectclasses = RT::ObjectClasses->new( $self->CurrentUser );
+    $objectclasses->LimitToClass( $self->Id );
+    $deps->Add( in => $topics );
+
+    # Custom Fields on things _in_ this class (CFs on the class itself
+    # have already been dealt with)
+    my $ocfs = RT::ObjectCustomFields->new( $self->CurrentUser );
+    $ocfs->Limit( FIELD           => 'ObjectId',
+                  OPERATOR        => '=',
+                  VALUE           => $self->id,
+                  ENTRYAGGREGATOR => 'OR' );
+    $ocfs->Limit( FIELD           => 'ObjectId',
+                  OPERATOR        => '=',
+                  VALUE           => 0,
+                  ENTRYAGGREGATOR => 'OR' );
+    my $cfs = $ocfs->Join(
+        ALIAS1 => 'main',
+        FIELD1 => 'CustomField',
+        TABLE2 => 'CustomFields',
+        FIELD2 => 'id',
+    );
+    $ocfs->Limit( ALIAS    => $cfs,
+                  FIELD    => 'LookupType',
+                  OPERATOR => 'STARTSWITH',
+                  VALUE    => 'RT::Class-' );
+    $deps->Add( in => $ocfs );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/CustomField.pm b/lib/RT/CustomField.pm
index 031fa9e..2127093 100644
--- a/lib/RT/CustomField.pm
+++ b/lib/RT/CustomField.pm
@@ -1988,6 +1988,22 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->BasedOnObj )
+        if $self->BasedOnObj->id;
+
+    my $applied = RT::ObjectCustomFields->new( $self->CurrentUser );
+    $applied->LimitToCustomField( $self->id );
+    $deps->Add( in => $applied );
+
+    $deps->Add( in => $self->Values ) if $self->ValuesClass eq "RT::CustomFieldValues";
+}
+
 
 RT::Base->_ImportOverlays();
 
diff --git a/lib/RT/CustomFieldValue.pm b/lib/RT/CustomFieldValue.pm
index ad39452..f733e55 100644
--- a/lib/RT/CustomFieldValue.pm
+++ b/lib/RT/CustomFieldValue.pm
@@ -329,6 +329,14 @@ sub _CoreAccessible {
 };
 
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->CustomFieldObj );
+}
 
 
 RT::Base->_ImportOverlays();
diff --git a/lib/RT/Group.pm b/lib/RT/Group.pm
index 1ee0591..9983ab1 100644
--- a/lib/RT/Group.pm
+++ b/lib/RT/Group.pm
@@ -1209,6 +1209,24 @@ sub PrincipalId {
     return $self->Id;
 }
 
+sub InstanceObj {
+    my $self = shift;
+
+    my $class;
+    if ( $self->Domain eq 'ACLEquivalence' ) {
+        $class = "RT::User";
+    } elsif ($self->Domain eq 'RT::Queue-Role') {
+        $class = "RT::Queue";
+    } elsif ($self->Domain eq 'RT::Ticket-Role') {
+        $class = "RT::Ticket";
+    }
+
+    return unless $class;
+
+    my $obj = $class->new( $self->CurrentUser );
+    $obj->Load( $self->Instance );
+    return $obj;
+}
 
 sub BasicColumns {
     (
@@ -1395,6 +1413,28 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    my $instance = $self->InstanceObj;
+    $deps->Add( out => $instance ) if $instance;
+
+    # Group members records, unless we're a system group
+    if ($self->Domain ne "SystemInternal") {
+        my $objs = RT::GroupMembers->new( $self->CurrentUser );
+        $objs->LimitToMembersOfGroup( $self->PrincipalId );
+        $deps->Add( in => $objs );
+    }
+
+    # Group member records group belongs to
+    my $objs = RT::GroupMembers->new( $self->CurrentUser );
+    $objs->Limit( FIELD => 'MemberId', VALUE => $self->PrincipalId );
+    $deps->Add( in => $objs );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/GroupMember.pm b/lib/RT/GroupMember.pm
index 84887ee..e03913e 100644
--- a/lib/RT/GroupMember.pm
+++ b/lib/RT/GroupMember.pm
@@ -487,6 +487,16 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->GroupObj->Object );
+    $deps->Add( out => $self->MemberObj->Object );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Link.pm b/lib/RT/Link.pm
index 2056f03..0ebc0f8 100644
--- a/lib/RT/Link.pm
+++ b/lib/RT/Link.pm
@@ -446,6 +446,16 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->BaseObj )   if $self->BaseObj   and $self->BaseObj->id;
+    $deps->Add( out => $self->TargetObj ) if $self->TargetObj and $self->TargetObj->id;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/ObjectClass.pm b/lib/RT/ObjectClass.pm
index c1d74cd..5901406 100644
--- a/lib/RT/ObjectClass.pm
+++ b/lib/RT/ObjectClass.pm
@@ -216,6 +216,19 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->ClassObj );
+
+    my $obj = $self->ObjectType->new( $self->CurrentUser );
+    $obj->Load( $self->ObjectId );
+    $deps->Add( out => $obj );
+}
+
 
 RT::Base->_ImportOverlays();
 
diff --git a/lib/RT/ObjectCustomField.pm b/lib/RT/ObjectCustomField.pm
index 45286ed..d3323a4 100644
--- a/lib/RT/ObjectCustomField.pm
+++ b/lib/RT/ObjectCustomField.pm
@@ -407,6 +407,20 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->CustomFieldObj );
+
+    my $class = $self->CustomFieldObj->RecordClassFromLookupType;
+    my $obj = $class->new( $self->CurrentUser );
+    $obj->Load( $self->ObjectId );
+    $deps->Add( out => $obj );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/ObjectCustomFieldValue.pm b/lib/RT/ObjectCustomFieldValue.pm
index c6c7882..90eaa43 100644
--- a/lib/RT/ObjectCustomFieldValue.pm
+++ b/lib/RT/ObjectCustomFieldValue.pm
@@ -751,6 +751,16 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->CustomFieldObj );
+    $deps->Add( out => $self->Object );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/ObjectTopic.pm b/lib/RT/ObjectTopic.pm
index e45b3e3..f7b2c19 100644
--- a/lib/RT/ObjectTopic.pm
+++ b/lib/RT/ObjectTopic.pm
@@ -209,6 +209,19 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->TopicObj );
+
+    my $obj = $self->ObjectType->new( $self->CurrentUser );
+    $obj->Load( $self->ObjectId );
+    $deps->Add( out => $obj );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Queue.pm b/lib/RT/Queue.pm
index 73733fc..819661c 100644
--- a/lib/RT/Queue.pm
+++ b/lib/RT/Queue.pm
@@ -1537,6 +1537,58 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    # Queue role groups( Cc, AdminCc )
+    my $objs = RT::Groups->new( $self->CurrentUser );
+    $objs->Limit( FIELD => 'Domain', VALUE => 'RT::Queue-Role' );
+    $objs->Limit( FIELD => 'Instance', VALUE => $self->Id );
+    $deps->Add( in => $objs );
+
+    # Scrips
+    $objs = RT::Scrips->new( $self->CurrentUser );
+    $objs->LimitToQueue( $self->id );
+    $deps->Add( in => $objs );
+
+    # Templates (global ones have already been dealt with)
+    $objs = RT::Templates->new( $self->CurrentUser );
+    $objs->Limit( FIELD => 'Queue', VALUE => $self->Id);
+    $deps->Add( in => $objs );
+
+    # Custom Fields on things _in_ this queue (CFs on the queue itself
+    # have already been dealt with)
+    $objs = RT::ObjectCustomFields->new( $self->CurrentUser );
+    $objs->Limit( FIELD           => 'ObjectId',
+                  OPERATOR        => '=',
+                  VALUE           => $self->id,
+                  ENTRYAGGREGATOR => 'OR' );
+    $objs->Limit( FIELD           => 'ObjectId',
+                  OPERATOR        => '=',
+                  VALUE           => 0,
+                  ENTRYAGGREGATOR => 'OR' );
+    my $cfs = $objs->Join(
+        ALIAS1 => 'main',
+        FIELD1 => 'CustomField',
+        TABLE2 => 'CustomFields',
+        FIELD2 => 'id',
+    );
+    $objs->Limit( ALIAS    => $cfs,
+                  FIELD    => 'LookupType',
+                  OPERATOR => 'STARTSWITH',
+                  VALUE    => 'RT::Queue-' );
+    $deps->Add( in => $objs );
+
+    # Tickets
+    $objs = RT::Tickets->new( $self->CurrentUser );
+    $objs->Limit( FIELD => 'Queue', VALUE => $self->Id );
+    $objs->{allow_deleted_search} = 1;
+    $deps->Add( in => $objs );
+}
+
 
 
 RT::Base->_ImportOverlays();
diff --git a/lib/RT/Scrip.pm b/lib/RT/Scrip.pm
index 5d4e5e8..f2a9d85 100644
--- a/lib/RT/Scrip.pm
+++ b/lib/RT/Scrip.pm
@@ -994,6 +994,18 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->ScripConditionObj );
+    $deps->Add( out => $self->ScripActionObj );
+    $deps->Add( out => $self->QueueObj );
+    $deps->Add( out => $self->TemplateObj );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Template.pm b/lib/RT/Template.pm
index 6f0251d..0cc2ee0 100644
--- a/lib/RT/Template.pm
+++ b/lib/RT/Template.pm
@@ -943,6 +943,15 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->QueueObj );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Ticket.pm b/lib/RT/Ticket.pm
index 316fbff..242c7af 100644
--- a/lib/RT/Ticket.pm
+++ b/lib/RT/Ticket.pm
@@ -4175,6 +4175,41 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    # Links
+    my $links = RT::Links->new( $self->CurrentUser );
+    $links->Limit(
+        SUBCLAUSE       => "either",
+        FIELD           => $_,
+        VALUE           => $self->URI,
+        ENTRYAGGREGATOR => 'OR'
+    ) for qw/Base Target/;
+    $deps->Add( in => $links );
+
+    # Tickets which were merged in
+    my $objs = RT::Tickets->new( $self->CurrentUser );
+    $objs->Limit( FIELD => 'EffectiveId', VALUE => $self->Id );
+    $objs->Limit( FIELD => 'id', OPERATOR => '!=', VALUE => $self->Id );
+    $deps->Add( in => $objs );
+
+    # Ticket role groups( Owner, Requestors, Cc, AdminCc )
+    $objs = RT::Groups->new( $self->CurrentUser );
+    $objs->Limit( FIELD => 'Domain', VALUE => 'RT::Ticket-Role' );
+    $objs->Limit( FIELD => 'Instance', VALUE => $self->Id );
+    $deps->Add( in => $objs );
+
+    # Queue
+    $deps->Add( out => $self->QueueObj );
+
+    # Owner
+    $deps->Add( out => $self->OwnerObj );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Topic.pm b/lib/RT/Topic.pm
index 34e4379..6ce74a4 100644
--- a/lib/RT/Topic.pm
+++ b/lib/RT/Topic.pm
@@ -372,5 +372,18 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+    $deps->Add( out => $self->ParentObj );
+    $deps->Add( in => $self->Children );
+
+    my $obj = $self->ObjectType->new( $self->CurrentUser );
+    $obj->Load( $self->ObjectId );
+    $deps->Add( out => $obj );
+}
+
 RT::Base->_ImportOverlays();
 1;
diff --git a/lib/RT/Transaction.pm b/lib/RT/Transaction.pm
index b032481..bc672e7 100644
--- a/lib/RT/Transaction.pm
+++ b/lib/RT/Transaction.pm
@@ -1608,6 +1608,59 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    $deps->Add( out => $self->Object );
+    $deps->Add( in => $self->Attachments );
+
+    my $type = $self->Type;
+    if ($type eq "CustomField") {
+        my $cf = RT::CustomField->new( RT->SystemUser );
+        $cf->Load( $self->Field );
+        $deps->Add( out => $cf );
+    } elsif ($type =~ /^(Take|Untake|Force|Steal|Give)$/) {
+        for my $field (qw/OldValue NewValue/) {
+            my $user = RT::User->new( RT->SystemUser );
+            $user->Load( $self->$field );
+            $deps->Add( out => $user );
+        }
+    } elsif ($type eq "DelWatcher") {
+        my $principal = RT::Principal->new( RT->SystemUser );
+        $principal->Load( $self->OldValue );
+        $deps->Add( out => $principal->Object );
+    } elsif ($type eq "AddWatcher") {
+        my $principal = RT::Principal->new( RT->SystemUser );
+        $principal->Load( $self->NewValue );
+        $deps->Add( out => $principal->Object );
+    } elsif ($type eq "DeleteLink") {
+        if ($self->OldValue) {
+            my $base = RT::URI->new( $self->CurrentUser );
+            $base->FromURI( $self->OldValue );
+            $deps->Add( out => $base->Object ) if $base->Resolver and $base->Object;
+        }
+    } elsif ($type eq "AddLink") {
+        if ($self->NewValue) {
+            my $base = RT::URI->new( $self->CurrentUser );
+            $base->FromURI( $self->NewValue );
+            $deps->Add( out => $base->Object ) if $base->Resolver and $base->Object;
+        }
+    } elsif ($type eq "Set" and $self->Field eq "Queue") {
+        for my $field (qw/OldValue NewValue/) {
+            my $queue = RT::Queue->new( RT->SystemUser );
+            $queue->Load( $self->$field );
+            $deps->Add( out => $queue );
+        }
+    } elsif ($type =~ /^(Add|Open|Resolve)Reminder$/) {
+        my $ticket = RT::Ticket->new( RT->SystemUser );
+        $ticket->Load( $self->NewValue );
+	$deps->Add( out => $ticket );
+    }
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/User.pm b/lib/RT/User.pm
index 9a82178..6abf679 100644
--- a/lib/RT/User.pm
+++ b/lib/RT/User.pm
@@ -2284,6 +2284,44 @@ sub _CoreAccessible {
  }
 };
 
+sub Dependencies {
+    my $self = shift;
+    my ($walker, $deps) = @_;
+
+    $self->SUPER::Dependencies($walker, $deps);
+
+    # ACL equivalence group
+    my $objs = RT::Groups->new( $self->CurrentUser );
+    $objs->Limit( FIELD => 'Domain', VALUE => 'ACLEquivalence' );
+    $objs->Limit( FIELD => 'Instance', VALUE => $self->Id );
+    $deps->Add( in => $objs );
+
+    # Memberships in SystemInternal groups
+    $objs = RT::GroupMembers->new( $self->CurrentUser );
+    $objs->Limit( FIELD => 'MemberId', VALUE => $self->Id );
+    my $principals = $objs->Join(
+        ALIAS1 => 'main',
+        FIELD1 => 'GroupId',
+        TABLE2 => 'Principals',
+        FIELD2 => 'id',
+    );
+    my $groups = $objs->Join(
+        ALIAS1 => $principals,
+        FIELD1 => 'ObjectId',
+        TABLE2 => 'Groups',
+        FIELD2 => 'Id',
+    );
+    $objs->Limit(
+        ALIAS => $groups,
+        FIELD => 'Domain',
+        VALUE => 'SystemInternal',
+    );
+    $deps->Add( in => $objs );
+
+    # XXX: This ignores the myriad of "in" references from the Creator
+    # and LastUpdatedBy columns.
+}
+
 RT::Base->_ImportOverlays();
 
 

commit 6f2d15ed91f4b831012ec0978b98596f07182bc4
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 00:00:10 2011 -0500

    Ensure that we always load merged tickets separately

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index 9df05c0..3059438 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -82,6 +82,16 @@ sub Walk {
 
     $self->PushObj( @_ );
 
+    # Ensure that RT::Ticket's ->Load doesn't follow a merged ticket to
+    # the ticket it was merged into.
+    no warnings 'redefine';
+    local *RT::Ticket::Load = sub {
+        my $self = shift;
+        my $id   = shift;
+        $self->LoadById( $id );
+        return $self->Id;
+    };
+
     $self->{visited} = {};
     $self->{seen}    = {};
 

commit 2a6cdd0b618131d249b4125c99c10bdf5ff9445d
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 01:15:48 2011 -0500

    Bare-bones serializer

diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
new file mode 100644
index 0000000..4057a65
--- /dev/null
+++ b/lib/RT/Serializer.pm
@@ -0,0 +1,142 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2011 Best Practical Solutions, LLC
+#                                          <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions,  LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+package RT::Serializer;
+
+use strict;
+use warnings;
+
+use base 'RT::DependencyWalker';
+
+use Storable qw//;
+use DateTime;
+
+sub Init {
+    my $self = shift;
+
+    $self->SUPER::Init(@_, First => "top");
+
+    # Keep track of the number of each type of object written out
+    $self->{ObjectCount} = {};
+}
+
+sub Walk {
+    my $self = shift;
+
+    # Set up our output file
+    open($self->{Filehandle}, ">", "serialized.dat"
+        or die "Can't write to serialized.dat: $!";
+
+    # Walk the objects
+    $self->SUPER::Walk( @_ );
+
+    # Close everything back up
+    close($self->{Filehandle})
+        or die "Can't close serialized.dat: $!";
+    $self->{FileCount}++;
+
+    return $self->ObjectCount;
+}
+
+sub ObjectCount {
+    my $self = shift;
+    return %{ $self->{ObjectCount} };
+}
+
+sub Observe {
+    my $self = shift;
+    my %args = (
+        object    => undef,
+        direction => undef,
+        from      => undef,
+        @_
+    );
+
+    my $obj = $args{object};
+    my $from = $args{from};
+    if ($obj->isa("RT::ACE")) {
+        return 0;
+    } elsif ($obj->isa("RT::GroupMember")) {
+        my $grp = $obj->GroupObj->Object;
+        if ($grp->Domain =~ /^RT::(Queue|Ticket)-Role$/) {
+            return 0 unless $grp->UID eq $from;
+        } elsif ($grp->Domain eq "SystemInternal") {
+            return 0 if $grp->UID eq $from;
+        }
+    } elsif ($obj->isa("RT::ObjectCustomField")) {
+        return 0 if $from =~ /^RT::CustomField-/;
+    }
+
+    return 1;
+}
+
+sub Visit {
+    my $self = shift;
+    my %args = (
+        object    => undef,
+        @_
+    );
+
+    # Serialize it
+    my $obj = $args{object};
+    my @store = (
+        ref($obj),
+        $obj->UID,
+        { $obj->Serialize },
+    );
+
+    # Write it out; nstore_fd doesn't trap failures to write, so we have
+    # to; by clearing $! and checking it afterwards.
+    $! = 0;
+    Storable::nstore_fd(\@store, $self->{Filehandle});
+    die "Failed to write to serialized.dat: $!" if $!;
+
+    $self->{ObjectCount}{ref($obj)}++;
+}
+
+1;
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
new file mode 100644
index 0000000..2799242
--- /dev/null
+++ b/sbin/rt-serializer
@@ -0,0 +1,22 @@
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+
+use lib 'lib';
+use RT;
+RT::LoadConfig();
+RT::Init();
+
+use RT::Serializer;
+my $walker = RT::Serializer->new();
+
+my $queues = RT::Queues->new( RT->SystemUser );
+$queues->UnLimit;
+$walker->PushObj( $queues );
+
+my %counts = $walker->Walk;
+
+print "Total object counts:\n";
+for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
+    printf "%8d %s\n", $counts{$_}, $_;
+}

commit f04b36bcf44fe7ee1d3a5a33c640e47519b062e4
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 01:17:53 2011 -0500

    Simplest Serialize method possible

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 494eab6..8e032e1 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -1985,6 +1985,17 @@ sub Dependencies {
     }
 }
 
+sub Serialize {
+    my $self = shift;
+
+    my %store = %{$self->{values}};
+
+    # Never store the ID
+    delete $store{id};
+
+    return %store;
+}
+
 RT::Base->_ImportOverlays();
 
 1;

commit 8ac005850611ef4230907df80e6913f5855542f7
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 01:19:30 2011 -0500

    Use _ClassAccessible and ->can to deduce possible inflated columns

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 8e032e1..4ffeb1d 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -1987,8 +1987,33 @@ sub Dependencies {
 
 sub Serialize {
     my $self = shift;
+    my %methods = (
+        Creator       => "CreatorObj",
+        LastUpdatedBy => "LastUpdatedByObj",
+        @_,
+    );
+
+    my %values = %{$self->{values}};
+    my %store;
+
+    my @cols = keys %{$self->_ClassAccessible};
+    @cols = grep {exists $values{lc $_} and defined $values{lc $_}} @cols;
+    for my $col ( @cols ) {
+        $store{$col} = $values{lc $col};
+        next unless $store{$col};
 
-    my %store = %{$self->{values}};
+        my $method = $methods{$col};
+        if (not $method) {
+            $method = $col;
+            $method =~ s/(Id)?$/Obj/;
+        }
+        next unless $self->can($method);
+
+        my $obj = $self->$method;
+        next unless $obj;
+        next unless $obj->isa("RT::Record");
+        $store{$col} = \($obj->UID);
+    }
 
     # Never store the ID
     delete $store{id};

commit c1450b40febc81b8d56fdc267c405c98dc74e91c
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 01:20:00 2011 -0500

    Serialize ObjectType and ObjectId into a single UID reference

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 4ffeb1d..a8e42f5 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -2018,6 +2018,12 @@ sub Serialize {
     # Never store the ID
     delete $store{id};
 
+    # Anything on an object should get the UID stored instead
+    if ($store{ObjectType} and $store{ObjectId}) {
+        delete $store{$_} for qw/ObjectType ObjectId/;
+        $store{Object} = \($self->Object->UID);
+    }
+
     return %store;
 }
 

commit eb49cf3db0e24c0b4564aec796ed111a3a9ff296
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 01:28:00 2011 -0500

    Slightly more comprehensive set of base objects to copy

diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index 2799242..aba5f4b 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -10,9 +10,32 @@ RT::Init();
 use RT::Serializer;
 my $walker = RT::Serializer->new();
 
+my $attributes = RT::System->new( RT->SystemUser )->Attributes;
+
+my $actions = RT::ScripActions->new( RT->SystemUser );
+$actions->UnLimit;
+
+my $conditions = RT::ScripConditions->new( RT->SystemUser );
+$conditions->UnLimit;
+
+my $users = RT::Users->new( RT->SystemUser );
+$users->LimitToPrivileged;
+
+my $groups = RT::Groups->new( RT->SystemUser );
+$groups->LimitToUserDefinedGroups;
+
+my $classes = RT::Classes->new( RT->SystemUser );
+$classes->UnLimit;
+
 my $queues = RT::Queues->new( RT->SystemUser );
 $queues->UnLimit;
-$walker->PushObj( $queues );
+
+$walker->PushObj(
+    $attributes,
+    $actions, $conditions,
+    $users, $groups,
+    $classes, $queues,
+);
 
 my %counts = $walker->Walk;
 

commit 35d21d706967f43f9667158132129a7499f73543
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 01:32:16 2011 -0500

    Output serialized data into bite-sized chunks

diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index 4057a65..04543ef 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -59,30 +59,85 @@ use DateTime;
 sub Init {
     my $self = shift;
 
+    my %args = (
+        Directory   => undef,
+        Force       => undef,
+        MaxFileSize => 32,
+        @_,
+    );
+
+    # Set up the output directory we'll be writing to
+    $args{Directory} = $RT::Organization . ":" . DateTime->now->ymd
+        unless defined $args{Directory};
+    system("rm", "-rf", $args{Directory}) if $args{Force};
+    die "Output directory $args{Directory} already exists"
+        if -d $args{Directory};
+    mkdir $args{Directory}
+        or die "Can't create output directory $args{Directory}: $!\n";
+    $self->{Directory} = delete $args{Directory};
+
+    # How many megabytes each chunk should be, approximitely
+    $self->{MaxFileSize} = delete $args{MaxFileSize};
+
     $self->SUPER::Init(@_, First => "top");
 
     # Keep track of the number of each type of object written out
     $self->{ObjectCount} = {};
+
+    # Which file we're writing to
+    $self->{FileCount} = 1;
 }
 
 sub Walk {
     my $self = shift;
 
     # Set up our output file
-    open($self->{Filehandle}, ">", "serialized.dat"
-        or die "Can't write to serialized.dat: $!";
+    open($self->{Filehandle}, ">", $self->Filename)
+        or die "Can't write to file @{[$self->Filename]}: $!";
+    push @{$self->{Files}}, $self->Filename;
 
     # Walk the objects
     $self->SUPER::Walk( @_ );
 
     # Close everything back up
     close($self->{Filehandle})
-        or die "Can't close serialized.dat: $!";
+        or die "Can't close @{[$self->Filename]}: $!";
     $self->{FileCount}++;
 
     return $self->ObjectCount;
 }
 
+sub Files {
+    my $self = shift;
+    return @{ $self->{Files} };
+}
+
+sub Filename {
+    my $self = shift;
+    return sprintf(
+        "%s/%03d.dat",
+        $self->{Directory},
+        $self->{FileCount}
+    );
+}
+
+sub Directory {
+    my $self = shift;
+    return $self->{Directory};
+}
+
+sub RotateFile {
+    my $self = shift;
+    close($self->{Filehandle})
+        or die "Can't close @{[$self->Filename]}: $!";
+    $self->{FileCount}++;
+
+    open($self->{Filehandle}, ">", $self->Filename)
+        or die "Can't write to file @{[$self->Filename]}: $!";
+
+    push @{$self->{Files}}, $self->Filename;
+}
+
 sub ObjectCount {
     my $self = shift;
     return %{ $self->{ObjectCount} };
@@ -122,6 +177,10 @@ sub Visit {
         @_
     );
 
+    # Rotate if we get too big
+    my $maxsize = 1024 * 1024 * $self->{MaxFileSize};
+    $self->RotateFile if tell($self->{Filehandle}) > $maxsize;
+
     # Serialize it
     my $obj = $args{object};
     my @store = (
@@ -134,7 +193,7 @@ sub Visit {
     # to; by clearing $! and checking it afterwards.
     $! = 0;
     Storable::nstore_fd(\@store, $self->{Filehandle});
-    die "Failed to write to serialized.dat: $!" if $!;
+    die "Failed to write to @{[$self->Filename]}: $!" if $!;
 
     $self->{ObjectCount}{ref($obj)}++;
 }
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index aba5f4b..87b6c96 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -7,8 +7,24 @@ use RT;
 RT::LoadConfig();
 RT::Init();
 
+use Getopt::Long;
+
+my %OPT;
+GetOptions(
+    \%OPT,
+
+    "directory|d=s",
+    "force!",
+    "size|s=i",
+);
+
+my %args;
+$args{Directory}   = $OPT{directory};
+$args{Force}       = $OPT{force};
+$args{MaxFileSize} = $OPT{size} if $OPT{size};
+
 use RT::Serializer;
-my $walker = RT::Serializer->new();
+my $walker = RT::Serializer->new( %args );
 
 my $attributes = RT::System->new( RT->SystemUser )->Attributes;
 
@@ -39,6 +55,11 @@ $walker->PushObj(
 
 my %counts = $walker->Walk;
 
+my @files = $walker->Files;
+print "Wrote @{[scalar @files]} files:\n";
+print "    $_\n" for @files;
+print "\n";
+
 print "Total object counts:\n";
 for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
     printf "%8d %s\n", $counts{$_}, $_;

commit 3c293e6cd39300799f2e866e8f9c00ff48840a4e
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 01:51:28 2011 -0500

    Serializing less than the whole db

diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index 04543ef..96a807a 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -63,6 +63,8 @@ sub Init {
         Directory   => undef,
         Force       => undef,
         MaxFileSize => 32,
+
+        FollowQueueToTicket => 0,
         @_,
     );
 
@@ -79,6 +81,8 @@ sub Init {
     # How many megabytes each chunk should be, approximitely
     $self->{MaxFileSize} = delete $args{MaxFileSize};
 
+    $self->{FollowQueueToTicket} = delete $args{FollowQueueToTicket};
+
     $self->SUPER::Init(@_, First => "top");
 
     # Keep track of the number of each type of object written out
@@ -154,7 +158,10 @@ sub Observe {
 
     my $obj = $args{object};
     my $from = $args{from};
-    if ($obj->isa("RT::ACE")) {
+    if ($obj->isa("RT::Ticket")) {
+        return $self->{FollowQueueToTicket}
+            if $from =~ /^RT::Queue-/;
+    } elsif ($obj->isa("RT::ACE")) {
         return 0;
     } elsif ($obj->isa("RT::GroupMember")) {
         my $grp = $obj->GroupObj->Object;
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index 87b6c96..988d0bb 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -16,42 +16,69 @@ GetOptions(
     "directory|d=s",
     "force!",
     "size|s=i",
+
+    "all|a!",
+    "queue|q=s",
+    "ticketsql|t=s",
 );
 
+die "Only one of --all, --queue, or --ticketsql allowed at once"
+    if (grep {$OPT{$_}} qw/all queue ticketsql/) > 1;
+
+$OPT{all} = 1 unless $OPT{queue} or $OPT{ticketsql};
+
+
 my %args;
 $args{Directory}   = $OPT{directory};
 $args{Force}       = $OPT{force};
 $args{MaxFileSize} = $OPT{size} if $OPT{size};
 
-use RT::Serializer;
-my $walker = RT::Serializer->new( %args );
-
-my $attributes = RT::System->new( RT->SystemUser )->Attributes;
 
-my $actions = RT::ScripActions->new( RT->SystemUser );
-$actions->UnLimit;
-
-my $conditions = RT::ScripConditions->new( RT->SystemUser );
-$conditions->UnLimit;
-
-my $users = RT::Users->new( RT->SystemUser );
-$users->LimitToPrivileged;
+unless ($OPT{ticketsql}) {
+    $args{FollowQueueToTicket} = 1;
+}
 
-my $groups = RT::Groups->new( RT->SystemUser );
-$groups->LimitToUserDefinedGroups;
 
-my $classes = RT::Classes->new( RT->SystemUser );
-$classes->UnLimit;
 
-my $queues = RT::Queues->new( RT->SystemUser );
-$queues->UnLimit;
+use RT::Serializer;
+my $walker = RT::Serializer->new( %args );
 
-$walker->PushObj(
-    $attributes,
-    $actions, $conditions,
-    $users, $groups,
-    $classes, $queues,
-);
+if ($OPT{all}) {
+    my $attributes = RT::System->new( RT->SystemUser )->Attributes;
+
+    my $actions = RT::ScripActions->new( RT->SystemUser );
+    $actions->UnLimit;
+
+    my $conditions = RT::ScripConditions->new( RT->SystemUser );
+    $conditions->UnLimit;
+
+    my $users = RT::Users->new( RT->SystemUser );
+    $users->LimitToPrivileged;
+
+    my $groups = RT::Groups->new( RT->SystemUser );
+    $groups->LimitToUserDefinedGroups;
+
+    my $classes = RT::Classes->new( RT->SystemUser );
+    $classes->UnLimit;
+
+    my $queues = RT::Queues->new( RT->SystemUser );
+    $queues->UnLimit;
+
+    $walker->PushObj(
+        $attributes,
+        $actions, $conditions,
+        $users, $groups,
+        $classes, $queues,
+    );
+} elsif ($OPT{queue}) {
+    my $queue = RT::Queue->new( RT->SystemUser );
+    $queue->Load( $OPT{queue} );
+    $walker->PushObj( $queue );
+} else {
+    my $tickets = RT::Tickets->new( RT->SystemUser );
+    $tickets->FromSQL( $OPT{ticketsql} );
+    $walker->PushObj( $tickets );
+}
 
 my %counts = $walker->Walk;
 

commit 802e7d21c4e6df4e006608920424b3651ad8efff
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 01:52:09 2011 -0500

    Serialize some global properties all of the time

diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index 96a807a..37b2e68 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -90,6 +90,55 @@ sub Init {
 
     # Which file we're writing to
     $self->{FileCount} = 1;
+
+    $self->PushBasics;
+}
+
+
+sub PushBasics {
+    my $self = shift;
+
+    # System users
+    for my $name (qw/RT_System root nobody/) {
+        my $user = RT::User->new( RT->SystemUser );
+        $user->Load( $name );
+        $self->PushObj( $user );
+    }
+
+    # System groups
+    foreach my $name (qw(Everyone Privileged Unprivileged)) {
+        my $group = RT::Group->new( RT->SystemUser );
+        $group->LoadSystemInternalGroup( $name );
+        $self->PushObj( $group );
+    }
+
+    # System role groups
+    my $systemroles = RT::Groups->new( RT->SystemUser );
+    $systemroles->LimitToRolesForSystem;
+    $self->PushObj( $systemroles );
+
+    # Global templates
+    my $templates = RT::Templates->new( RT->SystemUser );
+    $templates->LimitToGlobal;
+    $self->PushObj( $templates );
+
+    # Global scrips
+    my $scrips = RT::Scrips->new( RT->SystemUser );
+    $scrips->LimitToGlobal;
+    $self->PushObj( $scrips );
+
+    # CFs on Users, Groups, Queues
+    my $cfs = RT::CustomFields->new( RT->SystemUser );
+    $cfs->Limit(
+        FIELD => 'LookupType',
+        VALUE => $_
+    ) for qw/RT::User RT::Group RT::Queue/;
+    $self->PushObj( $cfs );
+
+    # Global topics
+    my $topics = RT::Topics->new( RT->SystemUser );
+    $topics->LimitToObject( RT::System->new );
+    $self->PushObj( $topics );
 }
 
 sub Walk {

commit eb81069bd3780d0b820fdc101ebd5d6a095b43e9
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 01:55:24 2011 -0500

    Better Serialize methods

diff --git a/lib/RT/CachedGroupMember.pm b/lib/RT/CachedGroupMember.pm
index c6ef008..55e9dd2 100644
--- a/lib/RT/CachedGroupMember.pm
+++ b/lib/RT/CachedGroupMember.pm
@@ -433,6 +433,10 @@ sub _CoreAccessible {
  }
 };
 
+sub Serialize {
+    die "CachedGroupMembers should never be serialized";
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Group.pm b/lib/RT/Group.pm
index 9983ab1..5d5fd3a 100644
--- a/lib/RT/Group.pm
+++ b/lib/RT/Group.pm
@@ -1435,6 +1435,18 @@ sub Dependencies {
     $deps->Add( in => $objs );
 }
 
+sub Serialize {
+    my $self = shift;
+    my %store = $self->SUPER::Serialize;
+
+    my $instance = $self->InstanceObj;
+    $store{Instance} = \($instance->UID) if $instance;
+
+    $store{Disabled} = $self->PrincipalObj->Disabled;
+    $store{Principal} = $self->PrincipalObj->UID;
+    return %store;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Link.pm b/lib/RT/Link.pm
index 0ebc0f8..aa66337 100644
--- a/lib/RT/Link.pm
+++ b/lib/RT/Link.pm
@@ -456,6 +456,15 @@ sub Dependencies {
     $deps->Add( out => $self->TargetObj ) if $self->TargetObj and $self->TargetObj->id;
 }
 
+sub Serialize {
+    my $self = shift;
+    my %store = $self->SUPER::Serialize;
+    delete $store{LocalBase}   if $store{Base};
+    delete $store{LocalTarget} if $store{Target};
+    return %store;
+}
+
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/ObjectCustomField.pm b/lib/RT/ObjectCustomField.pm
index d3323a4..40a1229 100644
--- a/lib/RT/ObjectCustomField.pm
+++ b/lib/RT/ObjectCustomField.pm
@@ -421,6 +421,20 @@ sub Dependencies {
     $deps->Add( out => $obj );
 }
 
+sub Serialize {
+    my $self = shift;
+    my %store = $self->SUPER::Serialize;
+
+    if ($store{ObjectId}) {
+        my $class = $self->CustomFieldObj->RecordClassFromLookupType;
+        my $obj = $class->new( RT->SystemUser );
+        $obj->Load( $store{ObjectId} );
+        $store{ObjectId} = \($obj->UID);
+    }
+    return %store;
+}
+
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Principal.pm b/lib/RT/Principal.pm
index 8046d78..4f3b6aa 100644
--- a/lib/RT/Principal.pm
+++ b/lib/RT/Principal.pm
@@ -817,6 +817,10 @@ sub _CoreAccessible {
  }
 };
 
+sub Serialize {
+    die "Principals should never be serialized";
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Ticket.pm b/lib/RT/Ticket.pm
index 242c7af..ce92d96 100644
--- a/lib/RT/Ticket.pm
+++ b/lib/RT/Ticket.pm
@@ -4210,6 +4210,17 @@ sub Dependencies {
     $deps->Add( out => $self->OwnerObj );
 }
 
+sub Serialize {
+    my $self = shift;
+    my %store = $self->SUPER::Serialize;
+
+    my $obj = RT::Ticket->new( RT->SystemUser );
+    $obj->Load( $store{EffectiveId} );
+    $store{EffectiveId} = \($obj->UID);
+
+    return %store;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Transaction.pm b/lib/RT/Transaction.pm
index bc672e7..f566673 100644
--- a/lib/RT/Transaction.pm
+++ b/lib/RT/Transaction.pm
@@ -1661,6 +1661,56 @@ sub Dependencies {
     }
 }
 
+sub Serialize {
+    my $self = shift;
+    my %store = $self->SUPER::Serialize;
+
+    my $type = $store{Type};
+    if ($type eq "CustomField") {
+        my $cf = RT::CustomField->new( RT->SystemUser );
+        $cf->Load( $store{Field} );
+        $store{Field} = \($cf->UID);
+    } elsif ($type =~ /^(Take|Untake|Force|Steal|Give)$/) {
+        for my $field (qw/OldValue NewValue/) {
+            my $user = RT::User->new( RT->SystemUser );
+            $user->Load( $store{$field} );
+            $store{$field} = \($user->UID);
+        }
+    } elsif ($type eq "DelWatcher") {
+        my $principal = RT::Principal->new( RT->SystemUser );
+        $principal->Load( $store{OldValue} );
+        $store{OldValue} = \($principal->UID);
+    } elsif ($type eq "AddWatcher") {
+        my $principal = RT::Principal->new( RT->SystemUser );
+        $principal->Load( $store{NewValue} );
+        $store{NewValue} = \($principal->UID);
+    } elsif ($type eq "DeleteLink") {
+        if ($store{OldValue}) {
+            my $base = RT::URI->new( $self->CurrentUser );
+            $base->FromURI( $store{OldValue} );
+            $store{OldValue} = \($base->Object->UID) if $base->Resolver and $base->Object;
+        }
+    } elsif ($type eq "AddLink") {
+        if ($store{NewValue}) {
+            my $base = RT::URI->new( $self->CurrentUser );
+            $base->FromURI( $store{NewValue} );
+            $store{NewValue} = \($base->Object->UID) if $base->Resolver and $base->Object;
+        }
+    } elsif ($type eq "Set" and $store{Field} eq "Queue") {
+        for my $field (qw/OldValue NewValue/) {
+            my $queue = RT::Queue->new( RT->SystemUser );
+            $queue->Load( $store{$field} );
+            $store{$field} = \($queue->UID);
+        }
+    } elsif ($type =~ /^(Add|Open|Resolve)Reminder$/) {
+        my $ticket = RT::Ticket->new( RT->SystemUser );
+        $ticket->Load( $store{NewValue} );
+        $store{NewValue} = \($ticket->UID);
+    }
+
+    return %store;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/User.pm b/lib/RT/User.pm
index 6abf679..2eab89f 100644
--- a/lib/RT/User.pm
+++ b/lib/RT/User.pm
@@ -2284,6 +2284,12 @@ sub _CoreAccessible {
  }
 };
 
+sub UID {
+    my $self = shift;
+    warn Carp::longmess("No Name for $self") unless defined $self->Name;
+    return "@{[ref $self]}-@{[$self->Name]}";
+}
+
 sub Dependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
@@ -2322,6 +2328,15 @@ sub Dependencies {
     # and LastUpdatedBy columns.
 }
 
+sub Serialize {
+    my $self = shift;
+    return (
+        Disabled => $self->PrincipalObj->Disabled,
+        Principal => $self->PrincipalObj->UID,
+        $self->SUPER::Serialize,
+    );
+}
+
 RT::Base->_ImportOverlays();
 
 

commit c488cca3179525406975f996128cb680bf996c36
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 02:05:58 2011 -0500

    Allow us to skip deleted tickets

diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index 37b2e68..78608f2 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -65,6 +65,7 @@ sub Init {
         MaxFileSize => 32,
 
         FollowQueueToTicket => 0,
+        FollowDeleted       => 1,
         @_,
     );
 
@@ -81,7 +82,10 @@ sub Init {
     # How many megabytes each chunk should be, approximitely
     $self->{MaxFileSize} = delete $args{MaxFileSize};
 
-    $self->{FollowQueueToTicket} = delete $args{FollowQueueToTicket};
+    $self->{$_} = delete $args{$_}
+        for qw/FollowQueueToTicket
+               FollowDeleted
+              /;
 
     $self->SUPER::Init(@_, First => "top");
 
@@ -208,6 +212,8 @@ sub Observe {
     my $obj = $args{object};
     my $from = $args{from};
     if ($obj->isa("RT::Ticket")) {
+        return $self->{FollowDeleted}
+            if $obj->Status eq "deleted";
         return $self->{FollowQueueToTicket}
             if $from =~ /^RT::Queue-/;
     } elsif ($obj->isa("RT::ACE")) {
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index 988d0bb..91ef384 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -34,7 +34,9 @@ $args{Force}       = $OPT{force};
 $args{MaxFileSize} = $OPT{size} if $OPT{size};
 
 
-unless ($OPT{ticketsql}) {
+if ($OPT{ticketsql}) {
+    $args{FollowDeleted} = 0;
+} else {
     $args{FollowQueueToTicket} = 1;
 }
 

commit ac6dedda7efcc6ba46eeafbf14525f4e0e96238c
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 02:09:17 2011 -0500

    Prepend groups and queues with the organization we were serialized from

diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index 78608f2..db38d6e 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -60,6 +60,7 @@ sub Init {
     my $self = shift;
 
     my %args = (
+        PostProcess => undef,
         Directory   => undef,
         Force       => undef,
         MaxFileSize => 32,
@@ -82,6 +83,10 @@ sub Init {
     # How many megabytes each chunk should be, approximitely
     $self->{MaxFileSize} = delete $args{MaxFileSize};
 
+    # An optional callback for doing arbitrary munging on data directly
+    # before it is written
+    $self->{PostProcess} = delete $args{PostProcess};
+
     $self->{$_} = delete $args{$_}
         for qw/FollowQueueToTicket
                FollowDeleted
@@ -250,6 +255,8 @@ sub Visit {
         $obj->UID,
         { $obj->Serialize },
     );
+    # Give callers one last chance to munge it before it hits disk
+    $self->{PostProcess}->(@store) if $self->{PostProcess};
 
     # Write it out; nstore_fd doesn't trap failures to write, so we have
     # to; by clearing $! and checking it afterwards.
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index 91ef384..b264588 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -40,6 +40,15 @@ if ($OPT{ticketsql}) {
     $args{FollowQueueToTicket} = 1;
 }
 
+$args{PostProcess} = sub {
+    my ($ref, $uid, $data) = @_;
+    if ($ref eq "RT::Group" and $data->{Domain} eq "UserDefined") {
+        $data->{Name} = $RT::Organization . ": " . $data->{Name};
+    } elsif ($ref eq "RT::Queue") {
+        $data->{Name} = $RT::Organization . ": " . $data->{Name}
+            unless $data->{Name} eq "___Approvals";
+    }
+};
 
 
 use RT::Serializer;

commit fa482f02293d870a7a826e0e887b01e96ac7e4b7
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 02:11:31 2011 -0500

    Provide --clone to copy everything, including ACLs, and not prepend $Organization

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index a8e42f5..0550248 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -1983,6 +1983,17 @@ sub Dependencies {
         $objs = $self->CustomFieldValues; # Actually OCFVs
         $deps->Add( in => $objs );
     }
+
+    # ACE records
+    if (   $self->isa("RT::Group")
+        or $self->isa("RT::Class")
+        or $self->isa("RT::Queue")
+        or $self->isa("RT::CustomField") )
+    {
+        $objs = RT::ACL->new( $self->CurrentUser );
+        $objs->LimitToObject( $self );
+        $deps->Add( in => $objs );
+    }
 }
 
 sub Serialize {
diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index db38d6e..65f254c 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -67,6 +67,7 @@ sub Init {
 
         FollowQueueToTicket => 0,
         FollowDeleted       => 1,
+        FollowACL           => 0,
         @_,
     );
 
@@ -90,6 +91,7 @@ sub Init {
     $self->{$_} = delete $args{$_}
         for qw/FollowQueueToTicket
                FollowDeleted
+               FollowACL
               /;
 
     $self->SUPER::Init(@_, First => "top");
@@ -222,7 +224,7 @@ sub Observe {
         return $self->{FollowQueueToTicket}
             if $from =~ /^RT::Queue-/;
     } elsif ($obj->isa("RT::ACE")) {
-        return 0;
+        return $self->{FollowACL};
     } elsif ($obj->isa("RT::GroupMember")) {
         my $grp = $obj->GroupObj->Object;
         if ($grp->Domain =~ /^RT::(Queue|Ticket)-Role$/) {
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index b264588..4cdb71d 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -17,14 +17,16 @@ GetOptions(
     "force!",
     "size|s=i",
 
+    "clone|c!",
     "all|a!",
     "queue|q=s",
     "ticketsql|t=s",
 );
 
-die "Only one of --all, --queue, or --ticketsql allowed at once"
-    if (grep {$OPT{$_}} qw/all queue ticketsql/) > 1;
+die "Only one of --clone, --all, --queue, or --ticketsql allowed at once"
+    if (grep {$OPT{$_}} qw/clone all queue ticketsql/) > 1;
 
+$OPT{all} = 1 if $OPT{clone};
 $OPT{all} = 1 unless $OPT{queue} or $OPT{ticketsql};
 
 
@@ -40,15 +42,19 @@ if ($OPT{ticketsql}) {
     $args{FollowQueueToTicket} = 1;
 }
 
-$args{PostProcess} = sub {
-    my ($ref, $uid, $data) = @_;
-    if ($ref eq "RT::Group" and $data->{Domain} eq "UserDefined") {
-        $data->{Name} = $RT::Organization . ": " . $data->{Name};
-    } elsif ($ref eq "RT::Queue") {
-        $data->{Name} = $RT::Organization . ": " . $data->{Name}
-            unless $data->{Name} eq "___Approvals";
-    }
-};
+if ($OPT{clone}) {
+    $args{FollowACL} = 1;
+} else {
+    $args{PostProcess} = sub {
+        my ($ref, $uid, $data) = @_;
+        if ($ref eq "RT::Group" and $data->{Domain} eq "UserDefined") {
+            $data->{Name} = $RT::Organization . ": " . $data->{Name};
+        } elsif ($ref eq "RT::Queue") {
+            $data->{Name} = $RT::Organization . ": " . $data->{Name}
+                unless $data->{Name} eq "___Approvals";
+        }
+    };
+}
 
 
 use RT::Serializer;

commit c3eb5bddbf6e755ccec028a43bcb5d81669da994
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 02:13:25 2011 -0500

    None, realtime, and every-n options for queue GC

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index 3059438..2131467 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -64,10 +64,12 @@ sub Init {
     my $self = shift;
     my %args = (
         First         => "top",
+        GC            => 0,
         @_
     );
 
     $self->{first}    = $args{First};
+    $self->{GC}       = $args{GC};
     $self->{stack}    = [];
 }
 
@@ -94,6 +96,7 @@ sub Walk {
 
     $self->{visited} = {};
     $self->{seen}    = {};
+    $self->{gc_count} = 0;
 
     my $stack = $self->{stack};
     while (@{$stack}) {
@@ -124,6 +127,25 @@ sub Walk {
         unshift @{$stack}, @{$self->{replace}};
         unshift @{$stack}, @{$self->{top}};
         push    @{$stack}, @{$self->{bottom}};
+
+        if ($self->{GC} > 0 and $self->{gc_count} > $self->{GC}) {
+            $self->{gc_count} = 0;
+            my $start_time = Time::HiRes::time();
+            my $start_size = @{$self->{stack}};
+            @{ $self->{stack} } = grep {
+                $_->{object}->isa("RT::Record")
+                    ? not exists $self->{visited}{$_->{uid} ||= $_->{object}->UID}
+                    : ( $_->{has_results} ||= do {
+                        $_->{object}->RowsPerPage(1);
+                        $_->{object}->Count;
+                    } )
+            } @{ $self->{stack} };
+            my $end_time = Time::HiRes::time();
+            my $end_size = @{$self->{stack}};
+            my $size = $start_size - $end_size;
+            my $time = $end_time - $start_time;
+            $self->{msg} = "GC -- $size removed, $time seconds, @{[$size/$time]}/s";
+        }
     }
 }
 
@@ -179,6 +201,7 @@ sub Process {
         }
         $obj->{satisfied}++;
         $self->{seen}{$uid}++;
+        $self->{gc_count}++ if $self->{GC} > 0;
     }
 }
 
@@ -197,7 +220,16 @@ sub AppendDeps {
     my $self = shift;
     my ($dir, $deps, $from) = @_;
     for my $obj (@{$deps->{$dir}}) {
-        next if $obj->isa("RT::Record") and not $obj->id;
+        if ($obj->isa("RT::Record")) {
+            next unless $obj->id;
+            next if $self->{GC} < 0 and exists $self->{seen}{$obj->UID};
+        } else {
+            $obj->FindAllRows;
+            if ($self->{GC} < 0) {
+                $obj->RowsPerPage(1);
+                next unless $obj->Count;
+            }
+        }
         push @{$self->{bottom}}, {
             object    => $obj,
             direction => $dir,
@@ -210,7 +242,16 @@ sub PrependDeps {
     my $self = shift;
     my ($dir, $deps, $from) = @_;
     for my $obj (@{$deps->{$dir}}) {
-        next if $obj->isa("RT::Record") and not $obj->id;
+        if ($obj->isa("RT::Record")) {
+            next unless $obj->id;
+            next if $self->{GC} < 0 and exists $self->{visited}{$obj->UID};
+        } else {
+            $obj->FindAllRows;
+            if ($self->{GC} < 0) {
+                $obj->RowsPerPage(1);
+                next unless $obj->Count;
+            }
+        }
         unshift @{$self->{top}}, {
             object    => $obj,
             direction => $dir,
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index 4cdb71d..b653b01 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -21,6 +21,8 @@ GetOptions(
     "all|a!",
     "queue|q=s",
     "ticketsql|t=s",
+
+    "gc=i",
 );
 
 die "Only one of --clone, --all, --queue, or --ticketsql allowed at once"
@@ -56,6 +58,7 @@ if ($OPT{clone}) {
     };
 }
 
+$args{GC} = $OPT{gc} if $OPT{gc};
 
 use RT::Serializer;
 my $walker = RT::Serializer->new( %args );

commit 7279afc0a1a298551ec8677fc47d4ab6c0f28c39
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 02:21:57 2011 -0500

    Basic importer

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
new file mode 100644
index 0000000..7441c1b
--- /dev/null
+++ b/lib/RT/Importer.pm
@@ -0,0 +1,161 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2011 Best Practical Solutions, LLC
+#                                          <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions,  LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+package RT::Importer;
+
+use strict;
+use warnings;
+
+use Storable qw//;
+
+sub new {
+    my $class = shift;
+    my $self = bless {}, $class;
+    $self->Init(@_);
+    return $self;
+}
+
+sub Init {
+    my $self = shift;
+    # Objects we've created
+    $self->{UIDs} = {};
+
+    # Columns we need to update when an object is later created
+    $self->{Pending} = {};
+
+    # What we created
+    $self->{ObjectCount} = {};
+}
+
+sub Resolve {
+    my $self = shift;
+    my ($uid, $class, $id) = @_;
+    $self->{UIDs}{$uid} = [ $class, $id ];
+    return unless $self->{Pending}{$uid};
+
+    for my $ref (@{$self->{Pending}{$uid}}) {
+        my ($pclass, $pid) = @{ $self->{UIDs}{ $ref->{uid} } };
+        my $obj = $pclass->new( RT->SystemUser );
+        $obj->LoadByCols( Id => $pid );
+        $obj->__Set(
+            Field => $ref->{column},
+            Value => $id,
+        );
+        $obj->__Set(
+            Field => $ref->{classcolumn},
+            Value => $class,
+        ) if $ref->{classcolumn};
+    }
+    delete $self->{Pending}{$uid};
+}
+
+sub Lookup {
+    my $self = shift;
+    my ($uid) = @_;
+    return $self->{UIDs}{$uid};
+}
+
+sub Postpone {
+    my $self = shift;
+    my %args = (
+        for         => undef,
+        uid         => undef,
+        column      => undef,
+        classcolumn => undef,
+        @_,
+    );
+    my $uid = delete $args{for};
+    push @{$self->{Pending}{$uid}}, \%args;
+}
+
+sub Import {
+    my $self = shift;
+    my @files = @_;
+
+    no warnings 'redefine';
+    local *RT::Ticket::Load = sub {
+        my $self = shift;
+        my $id   = shift;
+        $self->LoadById( $id );
+        return $self->Id;
+    };
+
+    for my $f (@files) {
+        open(my $fh, "<", $f) or die "Can't read $f: $!";
+        while (not eof($fh)) {
+            my $loaded = Storable::fd_retrieve($fh);
+            my ($class, $uid, $data) = @{$loaded};
+
+            next unless $class->PreInflate( $self, $uid, $data );
+
+            my $obj = $class->new( RT->SystemUser );
+            my ($id, $msg) = $obj->DBIx::SearchBuilder::Record::Create(
+                %{$data}
+            );
+            unless ($id) {
+                require Data::Dumper;
+                warn "Failed to create $uid: $msg\n" . Dumper($data);
+                next;
+            }
+
+            $self->{ObjectCount}{$class}++;
+            $self->Resolve( $uid => $class, $id );
+        }
+    }
+
+    # Anything we didn't see is an error
+    if (keys %{$self->{Pending}}) {
+        my @missing = sort keys %{$self->{Pending}};
+        warn "The following UIDs were expected but never observed: @missing";
+    }
+
+    # Return creation counts
+    return %{ $self->{ObjectCount} };
+}
+
+1;
diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 0550248..dd6d0bd 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -2038,6 +2038,52 @@ sub Serialize {
     return %store;
 }
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    my $ca = $class->_ClassAccessible;
+    my %ca = %{ $ca };
+    if ($data->{Object} and not $ca{Object}) {
+        my $ref_uid = ${ delete $data->{Object} };
+        my $ref = $importer->Lookup( $ref_uid );
+        if ($ref) {
+            my ($class, $id) = @{$ref};
+            $data->{ObjectId} = $id;
+            $data->{ObjectType} = $class;
+        } else {
+            $data->{ObjectId} = 0;
+            $data->{ObjectType} = "";
+            $importer->Postpone(
+                for => $ref_uid,
+                uid => $uid,
+                column => "ObjectId",
+                classcolumn => "ObjectType",
+            );
+        }
+    }
+
+    for my $col (keys %{$data}) {
+        if (ref $data->{$col}) {
+            my $ref_uid = ${ $data->{$col} };
+            my $ref = $importer->Lookup( $ref_uid );
+            if ($ref) {
+                my (undef, $id) = @{$ref};
+                $data->{$col} = $id;
+            } else {
+                $data->{$col} = 0;
+                $importer->Postpone(
+                    for => $ref_uid,
+                    uid => $uid,
+                    column => $col,
+                );
+            }
+        }
+    }
+
+    return 1;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/sbin/rt-importer b/sbin/rt-importer
new file mode 100644
index 0000000..e951a83
--- /dev/null
+++ b/sbin/rt-importer
@@ -0,0 +1,19 @@
+#!/usr/bin/perl -w
+use strict;
+use warnings;
+
+use lib 'lib';
+use RT;
+RT::LoadConfig();
+RT::Init();
+
+use RT::Importer;
+my $import = RT::Importer->new;
+
+my @files = map {-d $_ ? <$_/*.dat> : $_} @ARGV;
+my %counts = $import->Import( @files );
+
+print "Total object counts:\n";
+for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
+    printf "%8d %s\n", $counts{$_}, $_;
+}

commit f21a2d37d5545fe3eb110865e3e4290218ac25f5
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 02:30:36 2011 -0500

    Allow preservation of ticket IDs durimg import

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index 7441c1b..930756d 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -62,6 +62,14 @@ sub new {
 
 sub Init {
     my $self = shift;
+    my %args = (
+        PreserveTicketIds => 0,
+        @_,
+    );
+
+    # Should we attempt to preserve ticket IDs as they are created?
+    $self->{PreserveTicketIds} = $args{PreserveTicketIds};
+
     # Objects we've created
     $self->{UIDs} = {};
 
@@ -72,6 +80,11 @@ sub Init {
     $self->{ObjectCount} = {};
 }
 
+sub PreserveTicketIds {
+    my $self = shift;
+    return $self->{PreserveTicketIds};
+}
+
 sub Resolve {
     my $self = shift;
     my ($uid, $class, $id) = @_;
diff --git a/lib/RT/Ticket.pm b/lib/RT/Ticket.pm
index ce92d96..ecd0744 100644
--- a/lib/RT/Ticket.pm
+++ b/lib/RT/Ticket.pm
@@ -4218,9 +4218,21 @@ sub Serialize {
     $obj->Load( $store{EffectiveId} );
     $store{EffectiveId} = \($obj->UID);
 
+    # Shove the ID back in, in case we want to preserve it during import
+    $store{id} = $obj->Id;
+
     return %store;
 }
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    delete $data->{id} unless $importer->PreserveTicketIds;
+
+    return $class->SUPER::PreInflate( $importer, $uid, $data );
+}
+
 RT::Base->_ImportOverlays();
 
 1;

commit 7e7f86aa7d5fb6955a7dcab10f5ef2187cce944d
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 02:33:22 2011 -0500

    PreInflate for various object types

diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index c47efbd..95c2ebf 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -1043,6 +1043,24 @@ sub Dependencies {
     $deps->Add( out => $self->TransactionObj );
 }
 
+sub PreInflate {
+    my $class = shift;
+
+    my ($importer, $uid, $data) = @_;
+
+    if (defined $data->{Content}) {
+        my ($ContentEncoding, $Content) = $class->_EncodeLOB(
+            $data->{Content},
+            $data->{ContentType},
+            $data->{Filename}
+        );
+        $data->{ContentEncoding} = $ContentEncoding;
+        $data->{Content} = $Content;
+    }
+
+    return $class->SUPER::PreInflate( $importer, $uid, $data );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Class.pm b/lib/RT/Class.pm
index 29dc37a..9043024 100644
--- a/lib/RT/Class.pm
+++ b/lib/RT/Class.pm
@@ -658,6 +658,17 @@ sub Dependencies {
     $deps->Add( in => $ocfs );
 }
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    $class->SUPER::PreInflate( $importer, $uid, $data );
+
+    return if $importer->MergeBy( "Name", $class, $uid, $data );
+
+    return 1;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Group.pm b/lib/RT/Group.pm
index 5d5fd3a..07ab82d 100644
--- a/lib/RT/Group.pm
+++ b/lib/RT/Group.pm
@@ -1447,6 +1447,74 @@ sub Serialize {
     return %store;
 }
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    my $principal_uid = delete $data->{Principal};
+    my $disabled      = delete $data->{Disabled};
+
+    # Inflate refs into their IDs
+    $class->SUPER::PreInflate( $importer, $uid, $data );
+
+    # Factored out code, in case we find an existing version of this group
+    my $obj = RT::Group->new( RT->SystemUser );
+    my $duplicated = sub {
+        $importer->SkipTransactions( $uid );
+        $importer->Resolve(
+            $principal_uid,
+            ref($obj->PrincipalObj),
+            $obj->PrincipalObj->Id
+        );
+        $importer->Resolve( $uid => ref($obj), $obj->Id );
+        return;
+    };
+
+    # Go looking for the pre-existing version of the it
+    if ($data->{Domain} eq "ACLEquivalence") {
+        $obj->LoadACLEquivalenceGroup( $data->{Instance} );
+        if ($obj->Id) {
+            warn "---------- Skipping ACL equiv for existing user $data->{Instance}";
+            return $duplicated->();
+        }
+
+        # Update the name and description for the new ID
+        $data->{Name} = 'User '. $data->{Instance};
+        $data->{Description} = 'ACL equiv. for user '.$data->{Instance};
+    } elsif ($data->{Domain} eq "UserDefined") {
+        $obj->LoadUserDefinedGroup( $data->{Name} );
+        if ($obj->Id) {
+            warn "---------- Skipping user defined $data->{Name}";
+            $importer->MergeValues($obj, $data);
+            return $duplicated->();
+        }
+    } elsif ($data->{Domain} =~ /^(SystemInternal|RT::System-Role)$/) {
+        $obj->LoadByCols( Domain => $data->{Domain}, Type => $data->{Type} );
+        if ($obj->Id) {
+            warn "---------- Skipping $data->{Domain} $data->{Type}";
+            return $duplicated->();
+        }
+    } elsif ($data->{Domain} eq "RT::Queue-Role") {
+        $obj->LoadQueueRoleGroup( Queue => $data->{Instance}, Type => $data->{Type} );
+        if ($obj->Id) {
+            warn "---------- Skipping queue $data->{Instance} $data->{Type} role group";
+            return $duplicated->();
+        }
+    }
+
+    my $principal = RT::Principal->new( RT->SystemUser );
+    my ($id) = $principal->Create(
+        PrincipalType => 'Group',
+        Disabled => $disabled,
+        ObjectId => 0
+    );
+    $principal->__Set(Field => 'ObjectId', Value => $id);
+    $importer->Resolve( $principal_uid => ref($principal), $id );
+    $data->{id} = $id;
+
+    return 1;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/GroupMember.pm b/lib/RT/GroupMember.pm
index e03913e..c1e0db4 100644
--- a/lib/RT/GroupMember.pm
+++ b/lib/RT/GroupMember.pm
@@ -497,6 +497,26 @@ sub Dependencies {
     $deps->Add( out => $self->MemberObj->Object );
 }
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    $class->SUPER::PreInflate( $importer, $uid, $data );
+
+    my $obj = RT::GroupMember->new( RT->SystemUser );
+    $obj->LoadByCols(
+        GroupId  => $data->{GroupId},
+        MemberId => $data->{MemberId},
+    );
+    if ($obj->id) {
+        warn "............ Skipping existing membership $data->{MemberId} in $data->{GroupId}";
+        $importer->Resolve( $uid => ref($obj) => $obj->Id );
+        return;
+    }
+
+    return 1;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index 930756d..06883f3 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -113,6 +113,18 @@ sub Lookup {
     return $self->{UIDs}{$uid};
 }
 
+sub LookupObj {
+    my $self = shift;
+    my ($uid) = @_;
+    my $ref = $self->Lookup( $uid );
+    return unless $ref;
+    my ($class, $id) = @{ $ref };
+
+    my $obj = $class->new( RT->SystemUser );
+    $obj->Load( $id );
+    return $obj;
+}
+
 sub Postpone {
     my $self = shift;
     my %args = (
@@ -126,6 +138,53 @@ sub Postpone {
     push @{$self->{Pending}{$uid}}, \%args;
 }
 
+sub SkipTransactions {
+    my $self = shift;
+    my ($uid) = @_;
+    $self->{skiptransactions}{$uid} = 1;
+}
+
+sub ShouldSkipTransaction {
+    my $self = shift;
+    my ($uid) = @_;
+    return exists $self->{skiptransactions}{$uid};
+}
+
+sub MergeValues {
+    my $self = shift;
+    my ($obj, $data) = @_;
+    for my $col (keys %{$data}) {
+        next if defined $obj->__Value($col) and length $obj->__Value($col);
+        next unless defined $data->{$col} and length $data->{$col};
+        warn "$uid: Setting $col to $data->{$col}";
+        $obj->__Set( Field => $col, Value => $data->{$col} );
+    }
+}
+
+sub SkipBy {
+    my $self = shift;
+    my ($column, $class, $uid, $data) = @_;
+
+    my $obj = $class->new( RT->SystemUser );
+    $obj->Load( $data->{$column} );
+    return unless $obj->Id;
+
+    warn "!!!!!!!!!!!!!!!!!! Skipping $class @{[$obj->$column]}";
+    $self->SkipTransactions( $uid );
+
+    $self->Resolve( $uid => $class => $obj->Id );
+    return 1;
+}
+
+sub MergeBy {
+    my $self = shift;
+    my ($column, $class, $uid, $data) = @_;
+
+    return unless $self->SkipBy(@_);
+    $self->MergeValues( $obj, $data );
+    return 1;
+}
+
 sub Import {
     my $self = shift;
     my @files = @_;
diff --git a/lib/RT/Link.pm b/lib/RT/Link.pm
index aa66337..02e99dc 100644
--- a/lib/RT/Link.pm
+++ b/lib/RT/Link.pm
@@ -465,6 +465,23 @@ sub Serialize {
 }
 
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    for my $dir (qw/Base Target/) {
+        my $uid_ref = delete $data->{$dir};
+        next unless $uid_ref and ref $uid_ref;
+
+        my $uid = ${ $uid_ref };
+        my $obj = $importer->LookupObj( $uid );
+        $data->{$dir} = $obj->URI;
+        $data->{"Local$dir"} = $obj->Id if $obj->isa("RT::Ticket");
+    }
+
+    return $class->SUPER::PreInflate( $importer, $uid, $data );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/ObjectCustomFieldValue.pm b/lib/RT/ObjectCustomFieldValue.pm
index 90eaa43..208af5e 100644
--- a/lib/RT/ObjectCustomFieldValue.pm
+++ b/lib/RT/ObjectCustomFieldValue.pm
@@ -761,6 +761,23 @@ sub Dependencies {
     $deps->Add( out => $self->Object );
 }
 
+sub PreInflate {
+    my $class = shift;
+
+    my ($importer, $uid, $data) = @_;
+
+    if (defined $data->{LargeContent}) {
+        my ($ContentEncoding, $Content) = $class->_EncodeLOB(
+            $data->{LargeContent},
+            $data->{ContentType},
+        );
+        $data->{ContentEncoding} = $ContentEncoding;
+        $data->{LargeContent} = $Content;
+    }
+
+    return $class->SUPER::PreInflate( $importer, $uid, $data );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Queue.pm b/lib/RT/Queue.pm
index 819661c..0599dcc 100644
--- a/lib/RT/Queue.pm
+++ b/lib/RT/Queue.pm
@@ -1589,6 +1589,17 @@ sub Dependencies {
     $deps->Add( in => $objs );
 }
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    $class->SUPER::PreInflate( $importer, $uid, $data );
+
+    return if $importer->MergeBy( "Name", $class, $uid, $data );
+
+    return 1;
+}
+
 
 
 RT::Base->_ImportOverlays();
diff --git a/lib/RT/Scrip.pm b/lib/RT/Scrip.pm
index f2a9d85..7121fd1 100644
--- a/lib/RT/Scrip.pm
+++ b/lib/RT/Scrip.pm
@@ -1006,6 +1006,25 @@ sub Dependencies {
     $deps->Add( out => $self->TemplateObj );
 }
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    $class->SUPER::PreInflate( $importer, $uid, $data );
+
+    if ($data->{Queue} == 0) {
+        my $obj = RT::Scrip->new( RT->SystemUser );
+        $obj->LoadByCols( Queue => 0, Description => $data->{Description} );
+        if ($obj->Id) {
+            warn "---------- Skipping global scrip $data->{Description}";
+            $importer->Resolve( $uid => ref($obj) => $obj->Id );
+            return;
+        }
+    }
+
+    return 1;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/ScripAction.pm b/lib/RT/ScripAction.pm
index c679a91..4cbb333 100644
--- a/lib/RT/ScripAction.pm
+++ b/lib/RT/ScripAction.pm
@@ -411,6 +411,17 @@ sub _CoreAccessible {
  }
 };
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    $class->SUPER::PreInflate( $importer, $uid, $data );
+
+    return if $importer->SkipBy( "Name", $class, $uid, $data );
+
+    return 1;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/ScripCondition.pm b/lib/RT/ScripCondition.pm
index 3f24f2e..5c0b8ec 100644
--- a/lib/RT/ScripCondition.pm
+++ b/lib/RT/ScripCondition.pm
@@ -383,6 +383,17 @@ sub _CoreAccessible {
  }
 };
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    $class->SUPER::PreInflate( $importer, $uid, $data );
+
+    return if $importer->SkipBy( "Name", $class, $uid, $data );
+
+    return 1;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Template.pm b/lib/RT/Template.pm
index 0cc2ee0..886f002 100644
--- a/lib/RT/Template.pm
+++ b/lib/RT/Template.pm
@@ -952,6 +952,25 @@ sub Dependencies {
     $deps->Add( out => $self->QueueObj );
 }
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    $class->SUPER::PreInflate( $importer, $uid, $data );
+
+    if ($data->{Queue} == 0) {
+        my $obj = RT::Template->new( RT->SystemUser );
+        $obj->LoadGlobalTemplate( $data->{Name} );
+        if ($obj->Id) {
+            warn "---------- Skipping global template $data->{Name}";
+            $importer->Resolve( $uid => ref($obj) => $obj->Id );
+            return;
+        }
+    }
+
+    return 1;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Transaction.pm b/lib/RT/Transaction.pm
index f566673..d2f885c 100644
--- a/lib/RT/Transaction.pm
+++ b/lib/RT/Transaction.pm
@@ -1711,6 +1711,26 @@ sub Serialize {
     return %store;
 }
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    my $on_uid = ${ $data->{Object} };
+    return if $importer->ShouldSkipTransaction($on_uid);
+
+    if ($data->{Type} eq "DeleteLink" and ref $data->{OldValue}) {
+        my $uid = ${ $data->{OldValue} };
+        my $obj = $importer->LookupObj( $uid );
+        $data->{OldValue} = $obj->URI;
+    } elsif ($data->{Type} eq "AddLink" and ref $data->{NewValue}) {
+        my $uid = ${ $data->{NewValue} };
+        my $obj = $importer->LookupObj( $uid );
+        $data->{NewValue} = $obj->URI;
+    }
+
+    return $class->SUPER::PreInflate( $importer, $uid, $data );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/User.pm b/lib/RT/User.pm
index 2eab89f..0886f32 100644
--- a/lib/RT/User.pm
+++ b/lib/RT/User.pm
@@ -2337,6 +2337,46 @@ sub Serialize {
     );
 }
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    my $principal_uid = delete $data->{Principal};
+    my $disabled      = delete $data->{Disabled};
+
+    my $obj = RT::User->new( RT->SystemUser );
+    $obj->Load( $data->{Name} );
+
+    if ($obj->Id) {
+        # User already exists -- merge
+        warn "!!!!!!!!!!!!!!!!!! Merging @{[$obj->Name]}\n";
+        $importer->MergeValues($obj, $data);
+        $importer->SkipTransactions( $uid );
+
+        # Mark both the principal and the user object as resolved
+        $importer->Resolve(
+            $principal_uid,
+            ref($obj->PrincipalObj),
+            $obj->PrincipalObj->Id
+        );
+        $importer->Resolve( $uid => ref($obj) => $obj->Id );
+        return;
+    }
+
+    # Create a principal first, so we know what ID to use
+    my $principal = RT::Principal->new( RT->SystemUser );
+    my ($id) = $principal->Create(
+        PrincipalType => 'User',
+        Disabled => $disabled,
+        ObjectId => 0
+    );
+    $principal->__Set(Field => 'ObjectId', Value => $id);
+    $importer->Resolve( $principal_uid => ref($principal), $id );
+    $data->{id} = $id;
+
+    return $class->SUPER::PreInflate( $importer, $uid, $data );
+}
+
 RT::Base->_ImportOverlays();
 
 

commit eb2d46ffa1cd38182d84534951b978f5a92f5ad2
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 02:34:52 2011 -0500

    PostInflate, for columns which depend on our inserted ID

diff --git a/lib/RT/Article.pm b/lib/RT/Article.pm
index ca63c1d..33e46b7 100644
--- a/lib/RT/Article.pm
+++ b/lib/RT/Article.pm
@@ -871,6 +871,12 @@ sub Dependencies {
     $deps->Add( in => $self->Topics );
 }
 
+sub PostInflate {
+    my $self = shift;
+
+    $self->__Set( Field => 'URI', Value => $self->URI );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Group.pm b/lib/RT/Group.pm
index 07ab82d..ec25291 100644
--- a/lib/RT/Group.pm
+++ b/lib/RT/Group.pm
@@ -1515,6 +1515,17 @@ sub PreInflate {
     return 1;
 }
 
+sub PostInflate {
+    my $self = shift;
+
+    my $cgm = RT::CachedGroupMember->new($self->CurrentUser);
+    $cgm->Create(
+        Group  => $self->PrincipalObj,
+        Member => $self->PrincipalObj,
+        ImmediateParent => $self->PrincipalObj
+    );
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/GroupMember.pm b/lib/RT/GroupMember.pm
index c1e0db4..09c7413 100644
--- a/lib/RT/GroupMember.pm
+++ b/lib/RT/GroupMember.pm
@@ -95,6 +95,58 @@ Both Group and Member are expected to be RT::Principal objects
 
 =cut
 
+sub _InsertCGM {
+    my $self = shift;
+
+    my $cached_member = RT::CachedGroupMember->new( $self->CurrentUser );
+    my $cached_id     = $cached_member->Create(
+        Member          => $self->MemberObj,
+        Group           => $self->GroupObj,
+        ImmediateParent => $self->GroupObj,
+        Via             => '0'
+    );
+
+
+    #When adding a member to a group, we need to go back
+    #and popuplate the CachedGroupMembers of all the groups that group is part of .
+
+    my $cgm = RT::CachedGroupMembers->new( $self->CurrentUser );
+
+    # find things which have the current group as a member. 
+    # $group is an RT::Principal for the group.
+    $cgm->LimitToGroupsWithMember( $self->GroupId );
+    $cgm->Limit(
+        SUBCLAUSE => 'filter', # dont't mess up with prev condition
+        FIELD => 'MemberId',
+        OPERATOR => '!=',
+        VALUE => 'main.GroupId',
+        QUOTEVALUE => 0,
+        ENTRYAGGREGATOR => 'AND',
+    );
+
+    while ( my $parent_member = $cgm->Next ) {
+        my $parent_id = $parent_member->MemberId;
+        my $via       = $parent_member->Id;
+        my $group_id  = $parent_member->GroupId;
+
+        my $other_cached_member =
+            RT::CachedGroupMember->new( $self->CurrentUser );
+        my $other_cached_id = $other_cached_member->Create(
+            Member          => $self->MemberObj,
+                      Group => $parent_member->GroupObj,
+            ImmediateParent => $parent_member->MemberObj,
+            Via             => $parent_member->Id
+        );
+        unless ($other_cached_id) {
+            $RT::Logger->err( "Couldn't add " . $self->MemberId
+                  . " as a submember of a supergroup" );
+            return;
+        }
+    }
+
+    return $cached_id;
+}
+
 sub Create {
     my $self = shift;
     my %args = (
@@ -161,52 +213,9 @@ sub Create {
         return (undef);
     }
 
-    my $cached_member = RT::CachedGroupMember->new( $self->CurrentUser );
-    my $cached_id     = $cached_member->Create(
-        Member          => $args{'Member'},
-        Group           => $args{'Group'},
-        ImmediateParent => $args{'Group'},
-        Via             => '0'
-    );
-
-
-    #When adding a member to a group, we need to go back
-    #and popuplate the CachedGroupMembers of all the groups that group is part of .
-
-    my $cgm = RT::CachedGroupMembers->new( $self->CurrentUser );
-
-    # find things which have the current group as a member. 
-    # $group is an RT::Principal for the group.
-    $cgm->LimitToGroupsWithMember( $args{'Group'}->Id );
-    $cgm->Limit(
-        SUBCLAUSE => 'filter', # dont't mess up with prev condition
-        FIELD => 'MemberId',
-        OPERATOR => '!=',
-        VALUE => 'main.GroupId',
-        QUOTEVALUE => 0,
-        ENTRYAGGREGATOR => 'AND',
-    );
-
-    while ( my $parent_member = $cgm->Next ) {
-        my $parent_id = $parent_member->MemberId;
-        my $via       = $parent_member->Id;
-        my $group_id  = $parent_member->GroupId;
-
-          my $other_cached_member =
-          RT::CachedGroupMember->new( $self->CurrentUser );
-        my $other_cached_id = $other_cached_member->Create(
-            Member          => $args{'Member'},
-                      Group => $parent_member->GroupObj,
-            ImmediateParent => $parent_member->MemberObj,
-            Via             => $parent_member->Id
-        );
-        unless ($other_cached_id) {
-            $RT::Logger->err( "Couldn't add " . $args{'Member'}
-                  . " as a submember of a supergroup" );
-            $RT::Handle->Rollback() unless ($args{'InsideTransaction'});
-            return (undef);
-        }
-    } 
+    my $clone = RT::GroupMember->new( $self->CurrentUser );
+    $clone->Load( $id );
+    my $cached_id = $clone->_InsertCGM;
 
     unless ($cached_id) {
         $RT::Handle->Rollback() unless ($args{'InsideTransaction'});
@@ -517,6 +526,12 @@ sub PreInflate {
     return 1;
 }
 
+sub PostInflate {
+    my $self = shift;
+
+    $self->_InsertCGM;
+}
+
 RT::Base->_ImportOverlays();
 
 1;
diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index 06883f3..435328d 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -217,6 +217,11 @@ sub Import {
 
             $self->{ObjectCount}{$class}++;
             $self->Resolve( $uid => $class, $id );
+
+            # Load it back to get real values into the columns
+            $obj = $class->new( RT->SystemUser );
+            $obj->Load( $id );
+            $obj->PostInflate( $self );
         }
     }
 
diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index dd6d0bd..6e5dca2 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -2084,6 +2084,9 @@ sub PreInflate {
     return 1;
 }
 
+sub PostInflate {
+}
+
 RT::Base->_ImportOverlays();
 
 1;

commit 92d9abd89840b5195c2cd3c711c47cbd432b3741
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 02:35:42 2011 -0500

    Make global things not-global

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index 435328d..c1bed8d 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -197,12 +197,21 @@ sub Import {
         return $self->Id;
     };
 
+    my %unglobal;
+    my %new;
     for my $f (@files) {
         open(my $fh, "<", $f) or die "Can't read $f: $!";
         while (not eof($fh)) {
             my $loaded = Storable::fd_retrieve($fh);
             my ($class, $uid, $data) = @{$loaded};
 
+            # If it's a queue, store its ID away, as we'll need to know
+            # it to split global CFs into non-global across those
+            # fields.  We do this before inflating, so that queues which
+            # got merged still get the CFs applied
+            push @{$new{$class}}, $uid
+                if $class eq "RT::Queue";
+
             next unless $class->PreInflate( $self, $uid, $data );
 
             my $obj = $class->new( RT->SystemUser );
@@ -222,6 +231,25 @@ sub Import {
             $obj = $class->new( RT->SystemUser );
             $obj->Load( $id );
             $obj->PostInflate( $self );
+
+            # If it's a CF, we don't know yet if it's global (the OCF
+            # hasn't been created yet) to store away the CF for later
+            # inspection
+            push @{$unglobal{"RT::Queue"}}, $uid
+                if $class eq "RT::CustomField"
+                    and $obj->LookupType =~ /^RT::Queue/;
+        }
+    }
+
+    # Take global CFs which we made and make them un-global
+    for my $class (keys %unglobal) {
+        my @objs = grep {$_} map {$self->LookupObj( $_ )} @{$new{$class}};
+
+        for my $uid (@{$unglobal{$class}}) {
+            my $obj = $self->LookupObj( $uid );
+            my $ocf = $obj->IsApplied( 0 ) or next;
+            $ocf->Delete;
+            $obj->AddToObject( $_ ) for @objs;
         }
     }
 

commit ccb00a2f9f2fef09b08a194f4596df1da0dec3aa
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 02:36:49 2011 -0500

    Pretty graphs while you export

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index 2131467..1de324b 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -65,11 +65,13 @@ sub Init {
     my %args = (
         First         => "top",
         GC            => 0,
+        Progress      => undef,
         @_
     );
 
     $self->{first}    = $args{First};
     $self->{GC}       = $args{GC};
+    $self->{progress} = $args{Progress};
     $self->{stack}    = [];
 }
 
@@ -162,6 +164,7 @@ sub Process {
     return if $obj->isa("RT::System");
 
     my $uid = $obj->UID;
+    $self->{progress}->($obj) if $self->{progress};
     if (exists $self->{visited}{$uid}) {
         # Already visited -- no-op
         $self->Again(%args);
diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index 65f254c..f0c9f64 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -202,6 +202,11 @@ sub RotateFile {
     push @{$self->{Files}}, $self->Filename;
 }
 
+sub StackSize {
+    my $self = shift;
+    return scalar @{$self->{stack}};
+}
+
 sub ObjectCount {
     my $self = shift;
     return %{ $self->{ObjectCount} };
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index b653b01..d0d3a8f 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -17,6 +17,8 @@ GetOptions(
     "force!",
     "size|s=i",
 
+    "gnuplot|g!",
+
     "clone|c!",
     "all|a!",
     "queue|q=s",
@@ -37,6 +39,12 @@ $args{Directory}   = $OPT{directory};
 $args{Force}       = $OPT{force};
 $args{MaxFileSize} = $OPT{size} if $OPT{size};
 
+if ($OPT{gnuplot}) {
+    die "--gnuplot requires a gnuplot binary"
+        unless `which gnuplot`;
+    require Time::HiRes;
+    $args{Progress} = \&gnuplot;
+}
 
 if ($OPT{ticketsql}) {
     $args{FollowDeleted} = 0;
@@ -111,3 +119,64 @@ print "Total object counts:\n";
 for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
     printf "%8d %s\n", $counts{$_}, $_;
 }
+
+{
+    my $last_written;
+    my $last_time;
+    my $start;
+
+    sub plot {
+        my ($title, $column) = @_;
+
+        my ($col, $row) = (100, 50);
+        eval {
+            require Term::ReadKey;
+            ($col, $row) = Term::ReadKey::GetTerminalSize();
+        };
+        $col -= 1;
+        $row = int(($row - 5) / 2);
+
+        my $file = $walker->Directory . "/progress.plot";
+        system("gnuplot", "-e", <<EOT );
+set term dumb $col $row;
+set xlabel "Seconds";
+unset key;
+set autoscale;
+set title "$title";
+plot "$file" using 1:$column with lines
+EOT
+    }
+
+    sub gnuplot {
+        my $obj = shift;
+        my $now = Time::HiRes::time();
+        return if defined $last_time and $now - $last_time <= 3;
+
+        $start = $now unless $start;
+        my $length = $walker->StackSize;
+
+        my $written = 0;
+        my %counts = $walker->ObjectCount;
+        $written += $_ for values %counts;
+
+        open(my $dat, ">>", $walker->Directory . "/progress.plot");
+        printf $dat "%10.3f\t%8d\t%10.3f\n",
+            $now - $start,
+            $length,
+            defined $last_time ?
+                ($written - $last_written)/($now - $last_time) :
+                0;
+        close $dat;
+
+        if ($last_written) {
+            print `clear`;
+            plot( "Queue Length" => 2 );
+            plot( "Objects written per second" => 3 );
+            print $walker->{msg} . "\n" if $walker->{msg};
+            $walker->{msg} = "";
+        }
+
+        $last_written = $written;
+        $last_time = $now;
+    }
+}

commit 745fc09618f1a43cbefd86e8e41ba2ba0c108443
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 14:57:29 2011 -0500

    Syntax fixes

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index c1bed8d..f4356a7 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -156,7 +156,7 @@ sub MergeValues {
     for my $col (keys %{$data}) {
         next if defined $obj->__Value($col) and length $obj->__Value($col);
         next unless defined $data->{$col} and length $data->{$col};
-        warn "$uid: Setting $col to $data->{$col}";
+        warn $obj->UID . ": Setting $col to $data->{$col}";
         $obj->__Set( Field => $col, Value => $data->{$col} );
     }
 }
@@ -173,14 +173,15 @@ sub SkipBy {
     $self->SkipTransactions( $uid );
 
     $self->Resolve( $uid => $class => $obj->Id );
-    return 1;
+    return $obj;
 }
 
 sub MergeBy {
     my $self = shift;
     my ($column, $class, $uid, $data) = @_;
 
-    return unless $self->SkipBy(@_);
+    my $obj = $self->SkipBy(@_);
+    return unless $obj;
     $self->MergeValues( $obj, $data );
     return 1;
 }

commit becfe9398fefd83d3c7e9447284584dc9e769219
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 14:58:53 2011 -0500

    If we serialize RT::System attributes, we need to resolve it

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index f4356a7..6a5329d 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -198,6 +198,8 @@ sub Import {
         return $self->Id;
     };
 
+    $self->Resolve( RT->System->UID => ref RT->System, RT->System->Id );
+
     my %unglobal;
     my %new;
     for my $f (@files) {
diff --git a/lib/RT/System.pm b/lib/RT/System.pm
index 62289f0..2a7411d 100644
--- a/lib/RT/System.pm
+++ b/lib/RT/System.pm
@@ -211,6 +211,8 @@ Returns RT::System's id. It's 1.
 *Id = \&id;
 sub id { return 1 }
 
+sub UID { return "RT::System" }
+
 =head2 Load
 
 Since this object is pretending to be an RT::Record, we need a load method.

commit 4d161aeb865cd547ba452aef2b78172af0a8d967
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 15:00:36 2011 -0500

    Move PostProcess into object serialization instead

diff --git a/lib/RT/Group.pm b/lib/RT/Group.pm
index ec25291..1fe3b19 100644
--- a/lib/RT/Group.pm
+++ b/lib/RT/Group.pm
@@ -1442,6 +1442,9 @@ sub Serialize {
     my $instance = $self->InstanceObj;
     $store{Instance} = \($instance->UID) if $instance;
 
+    $store{Name} = "$RT::Organization: $store{Name}"
+        if $self->Domain eq "UserDefined";
+
     $store{Disabled} = $self->PrincipalObj->Disabled;
     $store{Principal} = $self->PrincipalObj->UID;
     return %store;
diff --git a/lib/RT/Queue.pm b/lib/RT/Queue.pm
index 0599dcc..f0627f8 100644
--- a/lib/RT/Queue.pm
+++ b/lib/RT/Queue.pm
@@ -1589,6 +1589,14 @@ sub Dependencies {
     $deps->Add( in => $objs );
 }
 
+sub Serialize {
+    my $self = shift;
+    my %store = $self->SUPER::Serialize;
+    $store{Name} = "$RT::Organization: $store{Name}"
+        if $self->Name ne "___Approvals";
+    return %store;
+}
+
 sub PreInflate {
     my $class = shift;
     my ($importer, $uid, $data) = @_;
diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index f0c9f64..4e8731e 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -60,7 +60,6 @@ sub Init {
     my $self = shift;
 
     my %args = (
-        PostProcess => undef,
         Directory   => undef,
         Force       => undef,
         MaxFileSize => 32,
@@ -84,10 +83,6 @@ sub Init {
     # How many megabytes each chunk should be, approximitely
     $self->{MaxFileSize} = delete $args{MaxFileSize};
 
-    # An optional callback for doing arbitrary munging on data directly
-    # before it is written
-    $self->{PostProcess} = delete $args{PostProcess};
-
     $self->{$_} = delete $args{$_}
         for qw/FollowQueueToTicket
                FollowDeleted
@@ -262,8 +257,6 @@ sub Visit {
         $obj->UID,
         { $obj->Serialize },
     );
-    # Give callers one last chance to munge it before it hits disk
-    $self->{PostProcess}->(@store) if $self->{PostProcess};
 
     # Write it out; nstore_fd doesn't trap failures to write, so we have
     # to; by clearing $! and checking it afterwards.
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index d0d3a8f..4e8fe0e 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -54,16 +54,6 @@ if ($OPT{ticketsql}) {
 
 if ($OPT{clone}) {
     $args{FollowACL} = 1;
-} else {
-    $args{PostProcess} = sub {
-        my ($ref, $uid, $data) = @_;
-        if ($ref eq "RT::Group" and $data->{Domain} eq "UserDefined") {
-            $data->{Name} = $RT::Organization . ": " . $data->{Name};
-        } elsif ($ref eq "RT::Queue") {
-            $data->{Name} = $RT::Organization . ": " . $data->{Name}
-                unless $data->{Name} eq "___Approvals";
-        }
-    };
 }
 
 $args{GC} = $OPT{gc} if $OPT{gc};

commit ce415e853d5ba535ecff309a67cd57c409ea4347
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 15:02:07 2011 -0500

    Move logic into RT::Importer rather than sbin/rt-importer

diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index 4e8731e..055e535 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -64,8 +64,12 @@ sub Init {
         Force       => undef,
         MaxFileSize => 32,
 
-        FollowQueueToTicket => 0,
+        AllUsers            => 1,
+        AllGroups           => 1,
         FollowDeleted       => 1,
+
+        FollowScrips        => 0,
+        FollowTickets       => 1,
         FollowACL           => 0,
         @_,
     );
@@ -84,9 +88,13 @@ sub Init {
     $self->{MaxFileSize} = delete $args{MaxFileSize};
 
     $self->{$_} = delete $args{$_}
-        for qw/FollowQueueToTicket
-               FollowDeleted
-               FollowACL
+        for qw/
+                  AllUsers
+                  AllGroups
+                  FollowDeleted
+                  FollowScrips
+                  FollowTickets
+                  FollowACL
               /;
 
     $self->SUPER::Init(@_, First => "top");
@@ -123,16 +131,6 @@ sub PushBasics {
     $systemroles->LimitToRolesForSystem;
     $self->PushObj( $systemroles );
 
-    # Global templates
-    my $templates = RT::Templates->new( RT->SystemUser );
-    $templates->LimitToGlobal;
-    $self->PushObj( $templates );
-
-    # Global scrips
-    my $scrips = RT::Scrips->new( RT->SystemUser );
-    $scrips->LimitToGlobal;
-    $self->PushObj( $scrips );
-
     # CFs on Users, Groups, Queues
     my $cfs = RT::CustomFields->new( RT->SystemUser );
     $cfs->Limit(
@@ -141,10 +139,48 @@ sub PushBasics {
     ) for qw/RT::User RT::Group RT::Queue/;
     $self->PushObj( $cfs );
 
-    # Global topics
+    # Global attributes
+    my $attributes = RT::System->new( RT->SystemUser )->Attributes;
+    $self->PushObj( $attributes );
+
+    # Global scrips
+    if ($self->{FollowScrips}) {
+        my $scrips = RT::Scrips->new( RT->SystemUser );
+        $scrips->LimitToGlobal;
+
+        my $templates = RT::Templates->new( RT->SystemUser );
+        $templates->LimitToGlobal;
+
+        my $actions = RT::ScripActions->new( RT->SystemUser );
+        $actions->UnLimit;
+
+        my $conditions = RT::ScripConditions->new( RT->SystemUser );
+        $conditions->UnLimit;
+        $self->PushObj( $scrips, $templates, $actions, $conditions );
+    }
+
+    if ($self->{AllUsers}) {
+        my $users = RT::Users->new( RT->SystemUser );
+        $users->LimitToPrivileged;
+        $self->PushObj( $users );
+    }
+
+    if ($self->{AllGroups}) {
+        my $groups = RT::Groups->new( RT->SystemUser );
+        $groups->LimitToUserDefinedGroups;
+        $self->PushObj( $groups );
+    }
+
     my $topics = RT::Topics->new( RT->SystemUser );
-    $topics->LimitToObject( RT::System->new );
-    $self->PushObj( $topics );
+    $topics->UnLimit;
+
+    my $classes = RT::Classes->new( RT->SystemUser );
+    $classes->UnLimit;
+    $self->PushObj( $topics, $classes );
+
+    my $queues = RT::Queues->new( RT->SystemUser );
+    $queues->UnLimit;
+    $self->PushObj( $queues );
 }
 
 sub Walk {
@@ -219,12 +255,12 @@ sub Observe {
     my $obj = $args{object};
     my $from = $args{from};
     if ($obj->isa("RT::Ticket")) {
-        return $self->{FollowDeleted}
-            if $obj->Status eq "deleted";
-        return $self->{FollowQueueToTicket}
-            if $from =~ /^RT::Queue-/;
+        return 0 if $obj->Status eq "deleted" and not $self->{FollowDeleted};
+        return $self->{FollowTickets};
     } elsif ($obj->isa("RT::ACE")) {
         return $self->{FollowACL};
+    } elsif ($obj->isa("RT::Scrip") or $obj->isa("RT::Template")) {
+        return $self->{FollowScrips};
     } elsif ($obj->isa("RT::GroupMember")) {
         my $grp = $obj->GroupObj->Object;
         if ($grp->Domain =~ /^RT::(Queue|Ticket)-Role$/) {
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index 4e8fe0e..7b09c45 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -19,21 +19,17 @@ GetOptions(
 
     "gnuplot|g!",
 
-    "clone|c!",
-    "all|a!",
-    "queue|q=s",
-    "ticketsql|t=s",
+    "users!",
+    "groups!",
+    "deleted!",
+
+    "scrips!",
+    "tickets!",
+    "acls!",
 
     "gc=i",
 );
 
-die "Only one of --clone, --all, --queue, or --ticketsql allowed at once"
-    if (grep {$OPT{$_}} qw/clone all queue ticketsql/) > 1;
-
-$OPT{all} = 1 if $OPT{clone};
-$OPT{all} = 1 unless $OPT{queue} or $OPT{ticketsql};
-
-
 my %args;
 $args{Directory}   = $OPT{directory};
 $args{Force}       = $OPT{force};
@@ -46,58 +42,19 @@ if ($OPT{gnuplot}) {
     $args{Progress} = \&gnuplot;
 }
 
-if ($OPT{ticketsql}) {
-    $args{FollowDeleted} = 0;
-} else {
-    $args{FollowQueueToTicket} = 1;
-}
+$args{AllUsers}      = $OPT{users}    if defined $OPT{users};
+$args{AllGroups}     = $OPT{groups}   if defined $OPT{groups};
+$args{FollowDeleted} = $OPT{deleted}  if defined $OPT{deleted};
 
-if ($OPT{clone}) {
-    $args{FollowACL} = 1;
-}
+$args{FollowScrips}  = $OPT{scrips}   if defined $OPT{scrips};
+$args{FollowTickets} = $OPT{tickets}  if defined $OPT{tickets};
+$args{FollowACL}     = $OPT{acls}     if defined $OPT{acls};
 
 $args{GC} = $OPT{gc} if $OPT{gc};
 
 use RT::Serializer;
 my $walker = RT::Serializer->new( %args );
 
-if ($OPT{all}) {
-    my $attributes = RT::System->new( RT->SystemUser )->Attributes;
-
-    my $actions = RT::ScripActions->new( RT->SystemUser );
-    $actions->UnLimit;
-
-    my $conditions = RT::ScripConditions->new( RT->SystemUser );
-    $conditions->UnLimit;
-
-    my $users = RT::Users->new( RT->SystemUser );
-    $users->LimitToPrivileged;
-
-    my $groups = RT::Groups->new( RT->SystemUser );
-    $groups->LimitToUserDefinedGroups;
-
-    my $classes = RT::Classes->new( RT->SystemUser );
-    $classes->UnLimit;
-
-    my $queues = RT::Queues->new( RT->SystemUser );
-    $queues->UnLimit;
-
-    $walker->PushObj(
-        $attributes,
-        $actions, $conditions,
-        $users, $groups,
-        $classes, $queues,
-    );
-} elsif ($OPT{queue}) {
-    my $queue = RT::Queue->new( RT->SystemUser );
-    $queue->Load( $OPT{queue} );
-    $walker->PushObj( $queue );
-} else {
-    my $tickets = RT::Tickets->new( RT->SystemUser );
-    $tickets->FromSQL( $OPT{ticketsql} );
-    $walker->PushObj( $tickets );
-}
-
 my %counts = $walker->Walk;
 
 my @files = $walker->Files;

commit 0671e606ca979ffa9eacda47f25385da050ae9f6
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 15:02:37 2011 -0500

    Add --verbose flag, for object-by-object updates

diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index 055e535..92619b8 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -71,6 +71,8 @@ sub Init {
         FollowScrips        => 0,
         FollowTickets       => 1,
         FollowACL           => 0,
+
+        Verbose => 1,
         @_,
     );
 
@@ -87,6 +89,8 @@ sub Init {
     # How many megabytes each chunk should be, approximitely
     $self->{MaxFileSize} = delete $args{MaxFileSize};
 
+    $self->{Verbose} = delete $args{Verbose};
+
     $self->{$_} = delete $args{$_}
         for qw/
                   AllUsers
@@ -288,6 +292,7 @@ sub Visit {
 
     # Serialize it
     my $obj = $args{object};
+    warn "Writing ".$obj->UID."\n" if $self->{Verbose};
     my @store = (
         ref($obj),
         $obj->UID,
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index 7b09c45..d290ee1 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -27,6 +27,7 @@ GetOptions(
     "tickets!",
     "acls!",
 
+    "verbose!",
     "gc=i",
 );
 
@@ -35,6 +36,8 @@ $args{Directory}   = $OPT{directory};
 $args{Force}       = $OPT{force};
 $args{MaxFileSize} = $OPT{size} if $OPT{size};
 
+$args{Verbose}     = $OPT{verbose};
+
 if ($OPT{gnuplot}) {
     die "--gnuplot requires a gnuplot binary"
         unless `which gnuplot`;

commit a2ebb0c088550df21b8e883b702b31dadbdf921f
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 15:57:03 2011 -0500

    Add command-line import option to preserve ticket IDs

diff --git a/sbin/rt-importer b/sbin/rt-importer
index e951a83..2a4cf61 100644
--- a/sbin/rt-importer
+++ b/sbin/rt-importer
@@ -7,8 +7,18 @@ use RT;
 RT::LoadConfig();
 RT::Init();
 
+use Getopt::Long;
+
+my %OPT;
+GetOptions(
+    \%OPT,
+    "preserve-tickets|t!",
+);
+
 use RT::Importer;
-my $import = RT::Importer->new;
+my $import = RT::Importer->new(
+    PreserveTicketIds => $OPT{'preserve-tickets'},
+);
 
 my @files = map {-d $_ ? <$_/*.dat> : $_} @ARGV;
 my %counts = $import->Import( @files );

commit dc1fd08b0067f6c540facd9c70ac564690f9a7db
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 16:13:37 2011 -0500

    Don't create dependent attributes if we found an equivilant object in the current DB

diff --git a/lib/RT/Attribute.pm b/lib/RT/Attribute.pm
index 101cc8f..f231186 100644
--- a/lib/RT/Attribute.pm
+++ b/lib/RT/Attribute.pm
@@ -635,6 +635,15 @@ sub Dependencies {
     $deps->Add( out => $self->Object );
 }
 
+sub PreInflate {
+    my $class = shift;
+    my ($importer, $uid, $data) = @_;
+
+    my $on_uid = ${ $data->{Object} };
+    return if $importer->ShouldSkipTransaction($on_uid);
+    return $class->SUPER::PreInflate( $importer, $uid, $data );
+}
+
 RT::Base->_ImportOverlays();
 
 1;

commit 8a6df07cfbc415e6bede276ff99954d40095bd06
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 16:14:02 2011 -0500

    Skip Attributes on the system object for now

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index 6a5329d..f7fcb3a 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -199,6 +199,7 @@ sub Import {
     };
 
     $self->Resolve( RT->System->UID => ref RT->System, RT->System->Id );
+    $self->SkipTransactions( RT->System->UID );
 
     my %unglobal;
     my %new;

commit 63716eac998416876629e9f2121fe62e40e7ba78
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 16:14:41 2011 -0500

    Warn on import if --preserve-tickets is passed and there are existing tickets

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index f7fcb3a..33d11c0 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -69,6 +69,10 @@ sub Init {
 
     # Should we attempt to preserve ticket IDs as they are created?
     $self->{PreserveTicketIds} = $args{PreserveTicketIds};
+    my $tickets = RT::Tickets->new( RT->SystemUser );
+    $tickets->UnLimit;
+    warn "There are already tickets in the system; preserving ticket IDs is unlikely to work"
+        if $tickets->Count;
 
     # Objects we've created
     $self->{UIDs} = {};

commit 163c3015a43df789a2563609707eb96bedfd270c
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 16:15:17 2011 -0500

    Tickets requires _SQLLimit rather than Limit

diff --git a/lib/RT/Queue.pm b/lib/RT/Queue.pm
index f0627f8..140dbcc 100644
--- a/lib/RT/Queue.pm
+++ b/lib/RT/Queue.pm
@@ -1584,7 +1584,7 @@ sub Dependencies {
 
     # Tickets
     $objs = RT::Tickets->new( $self->CurrentUser );
-    $objs->Limit( FIELD => 'Queue', VALUE => $self->Id );
+    $objs->_SQLLimit( FIELD => "Queue", VALUE => $self->Id );
     $objs->{allow_deleted_search} = 1;
     $deps->Add( in => $objs );
 }

commit 6983a345657c6b1b5aadec2f4885f5b7614abda2
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 16:15:36 2011 -0500

    Make sure we've loaded Time::HiRes prior to using it

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index 1de324b..3a3e0da 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -132,6 +132,7 @@ sub Walk {
 
         if ($self->{GC} > 0 and $self->{gc_count} > $self->{GC}) {
             $self->{gc_count} = 0;
+            require Time::HiRes;
             my $start_time = Time::HiRes::time();
             my $start_size = @{$self->{stack}};
             @{ $self->{stack} } = grep {

commit 6b3db4f28fd0679661a8a39b72d2e9cb925aa808
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 16:15:52 2011 -0500

    Warnings removal

diff --git a/lib/RT/Group.pm b/lib/RT/Group.pm
index 1fe3b19..e756696 100644
--- a/lib/RT/Group.pm
+++ b/lib/RT/Group.pm
@@ -1476,10 +1476,7 @@ sub PreInflate {
     # Go looking for the pre-existing version of the it
     if ($data->{Domain} eq "ACLEquivalence") {
         $obj->LoadACLEquivalenceGroup( $data->{Instance} );
-        if ($obj->Id) {
-            warn "---------- Skipping ACL equiv for existing user $data->{Instance}";
-            return $duplicated->();
-        }
+        return $duplicated->() if $obj->Id;
 
         # Update the name and description for the new ID
         $data->{Name} = 'User '. $data->{Instance};
@@ -1487,22 +1484,15 @@ sub PreInflate {
     } elsif ($data->{Domain} eq "UserDefined") {
         $obj->LoadUserDefinedGroup( $data->{Name} );
         if ($obj->Id) {
-            warn "---------- Skipping user defined $data->{Name}";
             $importer->MergeValues($obj, $data);
             return $duplicated->();
         }
     } elsif ($data->{Domain} =~ /^(SystemInternal|RT::System-Role)$/) {
         $obj->LoadByCols( Domain => $data->{Domain}, Type => $data->{Type} );
-        if ($obj->Id) {
-            warn "---------- Skipping $data->{Domain} $data->{Type}";
-            return $duplicated->();
-        }
+        return $duplicated->() if $obj->Id;
     } elsif ($data->{Domain} eq "RT::Queue-Role") {
         $obj->LoadQueueRoleGroup( Queue => $data->{Instance}, Type => $data->{Type} );
-        if ($obj->Id) {
-            warn "---------- Skipping queue $data->{Instance} $data->{Type} role group";
-            return $duplicated->();
-        }
+        return $duplicated->() if $obj->Id;
     }
 
     my $principal = RT::Principal->new( RT->SystemUser );
diff --git a/lib/RT/GroupMember.pm b/lib/RT/GroupMember.pm
index 09c7413..27dbf68 100644
--- a/lib/RT/GroupMember.pm
+++ b/lib/RT/GroupMember.pm
@@ -518,7 +518,6 @@ sub PreInflate {
         MemberId => $data->{MemberId},
     );
     if ($obj->id) {
-        warn "............ Skipping existing membership $data->{MemberId} in $data->{GroupId}";
         $importer->Resolve( $uid => ref($obj) => $obj->Id );
         return;
     }
diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index 33d11c0..5d01421 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -160,7 +160,6 @@ sub MergeValues {
     for my $col (keys %{$data}) {
         next if defined $obj->__Value($col) and length $obj->__Value($col);
         next unless defined $data->{$col} and length $data->{$col};
-        warn $obj->UID . ": Setting $col to $data->{$col}";
         $obj->__Set( Field => $col, Value => $data->{$col} );
     }
 }
@@ -173,7 +172,6 @@ sub SkipBy {
     $obj->Load( $data->{$column} );
     return unless $obj->Id;
 
-    warn "!!!!!!!!!!!!!!!!!! Skipping $class @{[$obj->$column]}";
     $self->SkipTransactions( $uid );
 
     $self->Resolve( $uid => $class => $obj->Id );
diff --git a/lib/RT/Scrip.pm b/lib/RT/Scrip.pm
index 7121fd1..2fd6a2f 100644
--- a/lib/RT/Scrip.pm
+++ b/lib/RT/Scrip.pm
@@ -1016,7 +1016,6 @@ sub PreInflate {
         my $obj = RT::Scrip->new( RT->SystemUser );
         $obj->LoadByCols( Queue => 0, Description => $data->{Description} );
         if ($obj->Id) {
-            warn "---------- Skipping global scrip $data->{Description}";
             $importer->Resolve( $uid => ref($obj) => $obj->Id );
             return;
         }
diff --git a/lib/RT/Template.pm b/lib/RT/Template.pm
index 886f002..f90369e 100644
--- a/lib/RT/Template.pm
+++ b/lib/RT/Template.pm
@@ -962,7 +962,6 @@ sub PreInflate {
         my $obj = RT::Template->new( RT->SystemUser );
         $obj->LoadGlobalTemplate( $data->{Name} );
         if ($obj->Id) {
-            warn "---------- Skipping global template $data->{Name}";
             $importer->Resolve( $uid => ref($obj) => $obj->Id );
             return;
         }
diff --git a/lib/RT/User.pm b/lib/RT/User.pm
index 0886f32..37413a6 100644
--- a/lib/RT/User.pm
+++ b/lib/RT/User.pm
@@ -2349,7 +2349,6 @@ sub PreInflate {
 
     if ($obj->Id) {
         # User already exists -- merge
-        warn "!!!!!!!!!!!!!!!!!! Merging @{[$obj->Name]}\n";
         $importer->MergeValues($obj, $data);
         $importer->SkipTransactions( $uid );
 

commit bfa7632a41385aa2c6d5eacc534613ce5dd8f5ce
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 18:14:09 2011 -0500

    Move PreserveTicketIds into the importer

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index 5d01421..f67bfc0 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -84,11 +84,6 @@ sub Init {
     $self->{ObjectCount} = {};
 }
 
-sub PreserveTicketIds {
-    my $self = shift;
-    return $self->{PreserveTicketIds};
-}
-
 sub Resolve {
     my $self = shift;
     my ($uid, $class, $id) = @_;
@@ -220,6 +215,10 @@ sub Import {
 
             next unless $class->PreInflate( $self, $uid, $data );
 
+            # Remove the ticket id, unless we specifically want it kept
+            delete $data->{id} if $class eq "RT::Ticket"
+                and not $self->{PreserveTicketIds};
+
             my $obj = $class->new( RT->SystemUser );
             my ($id, $msg) = $obj->DBIx::SearchBuilder::Record::Create(
                 %{$data}
diff --git a/lib/RT/Ticket.pm b/lib/RT/Ticket.pm
index ecd0744..fc72267 100644
--- a/lib/RT/Ticket.pm
+++ b/lib/RT/Ticket.pm
@@ -4224,15 +4224,6 @@ sub Serialize {
     return %store;
 }
 
-sub PreInflate {
-    my $class = shift;
-    my ($importer, $uid, $data) = @_;
-
-    delete $data->{id} unless $importer->PreserveTicketIds;
-
-    return $class->SUPER::PreInflate( $importer, $uid, $data );
-}
-
 RT::Base->_ImportOverlays();
 
 1;

commit af0fa410816b37d8a0b9587033cb32a8e9525c91
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 18:15:15 2011 -0500

    Add an option to add the original ticket ID as a CF

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index f67bfc0..9c3e55c 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -64,6 +64,7 @@ sub Init {
     my $self = shift;
     my %args = (
         PreserveTicketIds => 0,
+        OriginalId        => undef,
         @_,
     );
 
@@ -74,6 +75,8 @@ sub Init {
     warn "There are already tickets in the system; preserving ticket IDs is unlikely to work"
         if $tickets->Count;
 
+    $self->{OriginalId} = $args{OriginalId};
+
     # Objects we've created
     $self->{UIDs} = {};
 
@@ -237,6 +240,19 @@ sub Import {
             $obj->Load( $id );
             $obj->PostInflate( $self );
 
+            # If it's a ticket, we might need to create a
+            # TicketCustomField for the previous ID
+            if ($class eq "RT::Ticket" and $self->{OriginalId}) {
+                my ($org, $origid) = $uid =~ /^RT::Ticket-(.*)-(\d+)$/;
+                ($id, $msg) = $obj->AddCustomFieldValue(
+                    Field             => $self->{OriginalId},
+                    Value             => "$org:$origid",
+                    RecordTransaction => 0,
+                );
+                warn "Failed to add custom field to $uid: $msg"
+                    unless $id;
+            }
+
             # If it's a CF, we don't know yet if it's global (the OCF
             # hasn't been created yet) to store away the CF for later
             # inspection
diff --git a/sbin/rt-importer b/sbin/rt-importer
index 2a4cf61..4ba3554 100644
--- a/sbin/rt-importer
+++ b/sbin/rt-importer
@@ -13,13 +13,28 @@ my %OPT;
 GetOptions(
     \%OPT,
     "preserve-tickets|t!",
+    "originalid|i=s",
 );
 
 use RT::Importer;
 my $import = RT::Importer->new(
     PreserveTicketIds => $OPT{'preserve-tickets'},
+    OriginalId        => $OPT{originalid},
 );
 
+if ($OPT{originalid}) {
+    my $cf = RT::CustomField->new( RT->SystemUser );
+    $cf->LoadByName( Queue => 0, Name => $OPT{originalid} );
+    unless ($cf->Id) {
+        warn "Failed to find global CF named $OPT{originalid} -- creating one";
+        $cf->Create(
+            Queue => 0,
+            Name  => $OPT{originalid},
+            Type  => 'FreeformSingle',
+        );
+    }
+}
+
 my @files = map {-d $_ ? <$_/*.dat> : $_} @ARGV;
 my %counts = $import->Import( @files );
 

commit 81a5bd99b612e757c1fa5beb9eef7b5ca4001398
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 7 19:44:45 2011 -0500

    Only warn about existing tickets if we are attempting to preserve ticket ids

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index 9c3e55c..3d31e1d 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -70,10 +70,12 @@ sub Init {
 
     # Should we attempt to preserve ticket IDs as they are created?
     $self->{PreserveTicketIds} = $args{PreserveTicketIds};
-    my $tickets = RT::Tickets->new( RT->SystemUser );
-    $tickets->UnLimit;
-    warn "There are already tickets in the system; preserving ticket IDs is unlikely to work"
-        if $tickets->Count;
+    if ($self->{PreserveTicketIds}) {
+        my $tickets = RT::Tickets->new( RT->SystemUser );
+        $tickets->UnLimit;
+        warn "RT already contains tickets; preserving ticket IDs is unlikely to work"
+            if $tickets->Count;
+    }
 
     $self->{OriginalId} = $args{OriginalId};
 

commit 148de2ed05340096a6b97a98d558cac457bad209
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 8 03:39:23 2011 -0500

    Ensure that we export links to deleted tickets, as well

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index 3a3e0da..28241f4 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -96,6 +96,14 @@ sub Walk {
         return $self->Id;
     };
 
+    # When we walk ticket links, find deleted tickets as well
+    local *RT::Links::IsValidLink = sub {
+        my $self = shift;
+        my $link = shift;
+        return unless $link && ref $link && $link->Target && $link->Base;
+        return 1;
+    };
+
     $self->{visited} = {};
     $self->{seen}    = {};
     $self->{gc_count} = 0;

commit 8be5cb838d973710980057629e979a9b89bb95af
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 8 12:33:07 2011 -0500

    Write a summary file, to give importing a sense of what to expect

diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index 92619b8..dedf773 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -203,6 +203,14 @@ sub Walk {
         or die "Can't close @{[$self->Filename]}: $!";
     $self->{FileCount}++;
 
+    # Write the summary file
+    open(my $summary, ">", $self->Directory . "/summary");
+    Storable::nstore_fd({
+        files  => [ $self->Files ],
+        counts => { $self->ObjectCount },
+    }, $summary);
+    close($summary);
+
     return $self->ObjectCount;
 }
 

commit b9b9ee0d6da037b9f29b8f0902c959aa9b20bf1c
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 8 12:33:47 2011 -0500

    Major serializer UI tweaks (progress bars, drop write rate graph)

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index 28241f4..3bc0d09 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -66,12 +66,14 @@ sub Init {
         First         => "top",
         GC            => 0,
         Progress      => undef,
+        MessageHandler => sub { warn for @_ },
         @_
     );
 
     $self->{first}    = $args{First};
     $self->{GC}       = $args{GC};
     $self->{progress} = $args{Progress};
+    $self->{msg}      = $args{MessageHandler},
     $self->{stack}    = [];
 }
 
@@ -142,6 +144,7 @@ sub Walk {
             $self->{gc_count} = 0;
             require Time::HiRes;
             my $start_time = Time::HiRes::time();
+            $self->{msg}->("Starting GC pass...");
             my $start_size = @{$self->{stack}};
             @{ $self->{stack} } = grep {
                 $_->{object}->isa("RT::Record")
@@ -155,7 +158,7 @@ sub Walk {
             my $end_size = @{$self->{stack}};
             my $size = $start_size - $end_size;
             my $time = $end_time - $start_time;
-            $self->{msg} = "GC -- $size removed, $time seconds, @{[$size/$time]}/s";
+            $self->{msg}->("GC -- $size removed, $time seconds, @{[$size/$time]}/s");
         }
     }
 }
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index d290ee1..fdb072b 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -8,6 +8,7 @@ RT::LoadConfig();
 RT::Init();
 
 use Getopt::Long;
+use Time::HiRes qw//;
 
 my %OPT;
 GetOptions(
@@ -17,8 +18,6 @@ GetOptions(
     "force!",
     "size|s=i",
 
-    "gnuplot|g!",
-
     "users!",
     "groups!",
     "deleted!",
@@ -27,7 +26,6 @@ GetOptions(
     "tickets!",
     "acls!",
 
-    "verbose!",
     "gc=i",
 );
 
@@ -36,15 +34,6 @@ $args{Directory}   = $OPT{directory};
 $args{Force}       = $OPT{force};
 $args{MaxFileSize} = $OPT{size} if $OPT{size};
 
-$args{Verbose}     = $OPT{verbose};
-
-if ($OPT{gnuplot}) {
-    die "--gnuplot requires a gnuplot binary"
-        unless `which gnuplot`;
-    require Time::HiRes;
-    $args{Progress} = \&gnuplot;
-}
-
 $args{AllUsers}      = $OPT{users}    if defined $OPT{users};
 $args{AllGroups}     = $OPT{groups}   if defined $OPT{groups};
 $args{FollowDeleted} = $OPT{deleted}  if defined $OPT{deleted};
@@ -53,11 +42,24 @@ $args{FollowScrips}  = $OPT{scrips}   if defined $OPT{scrips};
 $args{FollowTickets} = $OPT{tickets}  if defined $OPT{tickets};
 $args{FollowACL}     = $OPT{acls}     if defined $OPT{acls};
 
-$args{GC} = $OPT{gc} if $OPT{gc};
+$args{GC} = defined $OPT{gc} ? $OPT{gc} : 5000;
+
+my $gnuplot = `which gnuplot`;
+my $msg = "";
+if (-t STDOUT) {
+    $args{Progress} = \&progress;
+    $args{MessageHandler} = sub {
+        print "\r", " "x60, "\r", $_[-1]; $msg = $_[-1];
+    };
+    $args{Verbose}  = 0;
+}
 
 use RT::Serializer;
 my $walker = RT::Serializer->new( %args );
 
+my %estimates = estimate();
+
+print "Beginning database serialization...";
 my %counts = $walker->Walk;
 
 my @files = $walker->Files;
@@ -70,63 +72,156 @@ for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
     printf "%8d %s\n", $counts{$_}, $_;
 }
 
+
+
+sub estimate {
+    $| = 1;
+    my %e;
+
+    # Expected types we'll serialize
+    my @types = qw/
+                      RT::Queue
+                      RT::Ticket
+                      RT::Transaction
+                      RT::Attachment
+                      RT::Link
+
+                      RT::User
+                      RT::Group
+                      RT::GroupMember
+
+                      RT::Attribute
+
+                      RT::CustomField
+                      RT::CustomFieldValue
+                      RT::ObjectCustomField
+                      RT::ObjectCustomFieldValue
+                  /;
+
+    $e{total} = 0;
+    for my $class (@types) {
+        print "Estimating $class count...";
+        my $collection = $class . "s";
+        my $objs = $collection->new( RT->SystemUser );
+        $objs->UnLimit;
+        $objs->{allow_deleted_search} = 1 if $class eq "RT::Ticket";
+        $e{$class} = $objs->Count;
+        $e{total} += $e{$class};
+        print "\r", " "x60, "\r";
+    }
+
+    return %e;
+}
+
+
 {
-    my $last_written;
     my $last_time;
     my $start;
+    my $left;
+    sub progress {
+        my $obj = shift;
+        my $now = Time::HiRes::time();
+        return if defined $last_time and $now - $last_time <= 3;
 
-    sub plot {
-        my ($title, $column) = @_;
+        $start = $now unless $start;
+        $last_time = $now;
 
-        my ($col, $row) = (100, 50);
+        my $elapsed = $now - $start;
+
+        print `clear`;
+        my ($cols, $rows) = (80, 25);
         eval {
             require Term::ReadKey;
-            ($col, $row) = Term::ReadKey::GetTerminalSize();
+            ($cols, $rows) = Term::ReadKey::GetTerminalSize();
         };
-        $col -= 1;
-        $row = int(($row - 5) / 2);
-
-        my $file = $walker->Directory . "/progress.plot";
-        system("gnuplot", "-e", <<EOT );
-set term dumb $col $row;
+        $cols -= 1;
+
+        if ($gnuplot) {
+            my $length = $walker->StackSize;
+            my $file = $walker->Directory . "/progress.plot";
+            open(my $dat, ">>", $file);
+            printf $dat "%10.3f\t%8d\n",
+                $elapsed,
+                $length;
+            close $dat;
+
+            if ($rows <= 24) {
+                print "\n\n";
+            } elsif ($elapsed) {
+                system("gnuplot", "-e", <<EOT );
+set term dumb $cols @{[$rows - 15]};
 set xlabel "Seconds";
 unset key;
 set autoscale;
-set title "$title";
-plot "$file" using 1:$column with lines
+set title "Queue length";
+plot "$file" using 1:2 with lines
 EOT
-    }
+            } else {
+                print "\n" for 1..($rows - 14);
+            }
+        } else {
+            print "\n\n";
+        }
 
-    sub gnuplot {
-        my $obj = shift;
-        my $now = Time::HiRes::time();
-        return if defined $last_time and $now - $last_time <= 3;
+        my %counts = $walker->ObjectCount;
+        # Just show the big ones
+        for my $class (map {"RT::$_"} qw/Ticket Transaction Attachment User Group/) {
+            my $display = $class;
+            $display =~ s/^RT::(.*)/@{[$1]}s:/;
 
-        $start = $now unless $start;
-        my $length = $walker->StackSize;
+            $counts{$class} ||= 0;
+            my $fraction = $counts{$class}/$estimates{$class};
 
-        my $written = 0;
-        my %counts = $walker->ObjectCount;
-        $written += $_ for values %counts;
-
-        open(my $dat, ">>", $walker->Directory . "/progress.plot");
-        printf $dat "%10.3f\t%8d\t%10.3f\n",
-            $now - $start,
-            $length,
-            defined $last_time ?
-                ($written - $last_written)/($now - $last_time) :
-                0;
-        close $dat;
-
-        if ($last_written) {
-            print `clear`;
-            plot( "Queue Length" => 2 );
-            plot( "Objects written per second" => 3 );
-            print $walker->{msg} . "\n" if $walker->{msg};
-            $walker->{msg} = "";
+            my $max_width = $cols - 30;
+            my $bar_width = int($max_width * $fraction);
+
+            printf "%20s |%-".$max_width."s|\n",
+                $display, "=" x $bar_width;
         }
 
-        $last_written = $written;
-        $last_time = $now;
+        {
+            my $total = 0;
+            $total += $_ for values %counts;
+            print "\n";
+
+            my $fraction = $total/$estimates{total};
+
+            my $max_width = $cols - 30;
+            my $bar_width = int($max_width * $fraction);
+
+            printf "%20s |%-".$max_width."s|\n",
+                "Total:", "#" x $bar_width;
+
+            if ($fraction > 0.03) {
+                if (defined $left) {
+                    $left = 0.75 * $left
+                          + 0.25 * ($elapsed / $fraction - $elapsed);
+                } else {
+                    $left = ($elapsed / $fraction - $elapsed);
+                }
+            }
+            print "\n";
+            printf "%20s %s\n", "Elapsed time:",
+                format_time($elapsed);
+            printf "%20s %s\n", "Estimated left:",
+                (defined $left) ? format_time($left) : "-";
+        }
+
+        print "\n$msg";
+        $msg = "";
     }
 }
+
+sub format_time {
+    my $time = shift;
+    my $s = "";
+
+    $s .= int($time/60/60)."hr "
+        if $time > 60*60;
+    $s .= int(($time % (60*60))/60)."min "
+        if $time > 60;
+    $s .= int($time % 60)."s"
+        if $time < 60*60;
+
+    return $s;
+}

commit 20e7c26eab0a415c894dc04349f71d285e828593
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 9 11:55:52 2011 -0500

    Trim some unnecessary decimal places from the GC message

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index 3bc0d09..acf9724 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -158,7 +158,12 @@ sub Walk {
             my $end_size = @{$self->{stack}};
             my $size = $start_size - $end_size;
             my $time = $end_time - $start_time;
-            $self->{msg}->("GC -- $size removed, $time seconds, @{[$size/$time]}/s");
+            $self->{msg}->(
+                sprintf(
+                    "GC -- %d removed, %.2f seconds, %d/s",
+                    $size, $time, int($size/$time)
+                )
+            );
         }
     }
 }

commit 927d2458849f2e75dfe47c2fb2c7b3bbb0edb7e4
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 9 11:57:25 2011 -0500

    Add rt-serializer POD, and --help

diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index fdb072b..fcea548 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -8,11 +8,13 @@ RT::LoadConfig();
 RT::Init();
 
 use Getopt::Long;
+use Pod::Usage qw//;
 use Time::HiRes qw//;
 
 my %OPT;
 GetOptions(
     \%OPT,
+    "help|?",
 
     "directory|d=s",
     "force!",
@@ -27,7 +29,9 @@ GetOptions(
     "acls!",
 
     "gc=i",
-);
+) or Pod::Usage::pod2usage();
+
+Pod::Usage::pod2usage(-verbose => 1) if $OPT{help};
 
 my %args;
 $args{Directory}   = $OPT{directory};
@@ -225,3 +229,114 @@ sub format_time {
 
     return $s;
 }
+
+=head1 NAME
+
+rt-serializer - Serialize an RT database to disk
+
+=head1 SYNOPSIS
+
+    rt-validator --check && rt-serializer
+
+This script is used to write out the entire RT database to disk, for
+later import into a different RT instance.  It requires that the data in
+the database be self-consistent, in order to do so; please make sure
+that the database being exported passes validation by L<rt-validator>
+before attempting to use C<rt-serializer>.
+
+While running, it will attempt to estimate the number of remaining
+objects to be serialized; these estimates are pessimistic, and will be
+incorrect if C<--no-users>, C<--no-groups>, or C<--no-tickets> are used.
+
+If the controlling terminal is large enough (more than 25 columns high)
+and the C<gnuplot> program is installed, it will also show a textual
+graph of the queue size over time.
+
+=head2 OPTIONS
+
+=over
+
+=item B<--directory> I<name>
+
+The name of the output directory to write data files to, which should
+not exist yet; it is a fatal error if it does.  Defaults to
+C<< ./I<$Organization>:I<Date>/ >>, where I<$Organization> is as set in
+F<RT_SiteConfig.pm>, and I<Date> is today's date.
+
+=item B<--force>
+
+Remove the output directory before starting.
+
+=item B<--size> I<megabytes>
+
+By default, C<rt-serializer> chunks its output into data files which are
+around 32Mb in size; this option is used to set a different threshold
+size, in megabytes.  Note that this is the threshold after which it
+rotates to writing a new file, and is as such the I<lower bound> on the
+size of each output file.
+
+=item B<--no-users>
+
+By default, all privileged users are serialized; passing C<--no-users>
+limits it to only those users which are strictly necessary.
+
+=item B<--no-groups>
+
+By default, all groups are serialized; passing C<--no-groups> limits it
+to only those groups which are strictly necessary.
+
+=item B<--no-deleted>
+
+By default, all tickets, including deleted tickets, are serialized;
+passing C<--no-deleted> skips deleted tickets during serialization.
+
+=item B<--scrips>
+
+No scrips or templates are serialized by default; this option forces all
+scrips and templates to be serialized.
+
+=item B<--acls>
+
+No ACLs are serialized by default; this option forces all ACLs to be
+serialized.
+
+=item B<--no-tickets>
+
+Skip serialization of all ticket data.
+
+=item B<--gc> I<n>
+
+Adjust how often the garbage collection sweep is done; lower numbers are
+more frequent.  See L</GARBAGE COLLECTION>.
+
+=back
+
+=head1 GARBAGE COLLECTION
+
+C<rt-serializer> maintains a priority queue of objects to serialize, or
+searches which may result in objects to serialize.  When inserting into
+this queue, it does no checking if the object in question is already in
+the queue, or if the search will contain any results.  These checks are
+done when the object reaches the front of the queue, or during periodic
+garbage collection.
+
+During periodic garbage collection, the entire queue is swept for
+objects which have already been serialized, occur more than once in the
+queue, and searches which contain no results in the database.  This is
+done to reduce the memory footprint of the serialization process, and is
+triggered when enough new objects have been placed in the queue.  This
+parameter is tunable via the C<--gc> parameter, which defaults to
+running garbage collection every 5,000 objects inserted into the queue;
+smaller numbers will result in more frequent garbage collection.
+
+The default of 5,000 is roughly tuned based on a database with several
+thousand tickets, but optimal values will vary wildly depending on
+database configuration and size.  Values as low as 25 have provided
+speedups with smaller databases; if speed is a factor, experimenting
+with different C<--gc> values may be helpful.  Note that there are
+significant boundry condition changes in serialization rate, as the
+queue empties and fills, causing the time estimates to be rather
+imprecise near the start and end of the process.
+
+=cut
+

commit 3ca56a79189d2df7891cac7f33c3dab7469dbeb9
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 9 16:43:04 2011 -0500

    Move CF creation into RT::Importer

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index 3d31e1d..e8a6d94 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -77,7 +77,20 @@ sub Init {
             if $tickets->Count;
     }
 
+    # Where to shove the original ticket ID
     $self->{OriginalId} = $args{OriginalId};
+    if ($self->{OriginalId}) {
+        my $cf = RT::CustomField->new( RT->SystemUser );
+        $cf->LoadByName( Queue => 0, Name => $self->{OriginalId} );
+        unless ($cf->Id) {
+            warn "Failed to find global CF named $self->{OriginalId} -- creating one";
+            $cf->Create(
+                Queue => 0,
+                Name  => $self->{OriginalId},
+                Type  => 'FreeformSingle',
+            );
+        }
+    }
 
     # Objects we've created
     $self->{UIDs} = {};
diff --git a/sbin/rt-importer b/sbin/rt-importer
index 4ba3554..5c25c46 100644
--- a/sbin/rt-importer
+++ b/sbin/rt-importer
@@ -21,20 +21,6 @@ my $import = RT::Importer->new(
     PreserveTicketIds => $OPT{'preserve-tickets'},
     OriginalId        => $OPT{originalid},
 );
-
-if ($OPT{originalid}) {
-    my $cf = RT::CustomField->new( RT->SystemUser );
-    $cf->LoadByName( Queue => 0, Name => $OPT{originalid} );
-    unless ($cf->Id) {
-        warn "Failed to find global CF named $OPT{originalid} -- creating one";
-        $cf->Create(
-            Queue => 0,
-            Name  => $OPT{originalid},
-            Type  => 'FreeformSingle',
-        );
-    }
-}
-
 my @files = map {-d $_ ? <$_/*.dat> : $_} @ARGV;
 my %counts = $import->Import( @files );
 

commit 872983d229f31afd226a9e8c3ff9115593461bba
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 9 16:44:06 2011 -0500

    Use the more straightforward nstore function, rather than opening the file ourselves

diff --git a/lib/RT/Serializer.pm b/lib/RT/Serializer.pm
index dedf773..a8d6981 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Serializer.pm
@@ -204,12 +204,10 @@ sub Walk {
     $self->{FileCount}++;
 
     # Write the summary file
-    open(my $summary, ">", $self->Directory . "/summary");
-    Storable::nstore_fd({
+    Storable::nstore( {
         files  => [ $self->Files ],
         counts => { $self->ObjectCount },
-    }, $summary);
-    close($summary);
+    }, $self->Directory . "/rt-serialized");
 
     return $self->ObjectCount;
 }

commit ea3f8939e0c6a87725538f2cce95891ab06e6f36
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 9 16:44:45 2011 -0500

    Split out ObjectCount function, as in RT::DependencyWalker

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index e8a6d94..9c04b44 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -296,6 +296,11 @@ sub Import {
     }
 
     # Return creation counts
+    return $self->ObjectCount;
+}
+
+sub ObjectCount {
+    my $self = shift;
     return %{ $self->{ObjectCount} };
 }
 

commit 87192807392c726a799195eba68c12886195cd1c
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 9 16:45:13 2011 -0500

    Add hooks for a progress callback in the importer, as well

diff --git a/lib/RT/Importer.pm b/lib/RT/Importer.pm
index 9c04b44..6bde71a 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Importer.pm
@@ -65,6 +65,7 @@ sub Init {
     my %args = (
         PreserveTicketIds => 0,
         OriginalId        => undef,
+        Progress          => undef,
         @_,
     );
 
@@ -92,6 +93,8 @@ sub Init {
         }
     }
 
+    $self->{Progress} = $args{Progress};
+
     # Objects we've created
     $self->{UIDs} = {};
 
@@ -274,6 +277,8 @@ sub Import {
             push @{$unglobal{"RT::Queue"}}, $uid
                 if $class eq "RT::CustomField"
                     and $obj->LookupType =~ /^RT::Queue/;
+
+            $self->{Progress}->($obj) if $self->{Progress};
         }
     }
 

commit 526beaeb4fa1d8cdbf3432e74402ce8f4de66fbb
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 9 16:46:19 2011 -0500

    Progressbars on the importer, based on the summary data

diff --git a/sbin/rt-importer b/sbin/rt-importer
index 5c25c46..f2f43c8 100644
--- a/sbin/rt-importer
+++ b/sbin/rt-importer
@@ -8,6 +8,7 @@ RT::LoadConfig();
 RT::Init();
 
 use Getopt::Long;
+use Time::HiRes qw//;
 
 my %OPT;
 GetOptions(
@@ -16,10 +17,21 @@ GetOptions(
     "originalid|i=s",
 );
 
+my %estimates;
+my ($summary) = grep {-f $_} map {"$_/rt-serialized"} @ARGV;
+if ($summary) {
+    my $ref = Storable::retrieve($summary);
+    %estimates = %{ $ref->{counts} };
+    my $total = 0;
+    $total += $_ for values %estimates;
+    $estimates{total} = $total;
+}
+
 use RT::Importer;
 my $import = RT::Importer->new(
     PreserveTicketIds => $OPT{'preserve-tickets'},
     OriginalId        => $OPT{originalid},
+    Progress          => (%estimates ? \&progress : undef),
 );
 my @files = map {-d $_ ? <$_/*.dat> : $_} @ARGV;
 my %counts = $import->Import( @files );
@@ -28,3 +40,86 @@ print "Total object counts:\n";
 for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
     printf "%8d %s\n", $counts{$_}, $_;
 }
+
+
+{
+    my $last_time;
+    my $start;
+    my $left;
+    sub progress {
+        my $obj = shift;
+        my $now = Time::HiRes::time();
+        return if defined $last_time and $now - $last_time <= 3;
+
+        $start = $now unless $start;
+        $last_time = $now;
+
+        my $elapsed = $now - $start;
+
+        print `clear`;
+        my ($cols, $rows) = (80, 25);
+        eval {
+            require Term::ReadKey;
+            ($cols, $rows) = Term::ReadKey::GetTerminalSize();
+        };
+        $cols -= 1;
+
+        my %counts = $import->ObjectCount;
+        # Just show the big ones
+        for my $class (map {"RT::$_"} qw/Ticket Transaction Attachment User Group/) {
+            my $display = $class;
+            $display =~ s/^RT::(.*)/@{[$1]}s:/;
+
+            $counts{$class} ||= 0;
+            my $fraction = $counts{$class}/$estimates{$class};
+
+            my $max_width = $cols - 30;
+            my $bar_width = int($max_width * $fraction);
+
+            printf "%20s |%-".$max_width."s|\n",
+                $display, "=" x $bar_width;
+        }
+
+        {
+            my $total = 0;
+            $total += $_ for values %counts;
+            print "\n";
+
+            my $fraction = $total/$estimates{total};
+
+            my $max_width = $cols - 30;
+            my $bar_width = int($max_width * $fraction);
+
+            printf "%20s |%-".$max_width."s|\n",
+                "Total:", "#" x $bar_width;
+
+            if ($fraction > 0.03) {
+                if (defined $left) {
+                    $left = 0.75 * $left
+                          + 0.25 * ($elapsed / $fraction - $elapsed);
+                } else {
+                    $left = ($elapsed / $fraction - $elapsed);
+                }
+            }
+            print "\n";
+            printf "%20s %s\n", "Elapsed time:",
+                format_time($elapsed);
+            printf "%20s %s\n", "Estimated left:",
+                (defined $left) ? format_time($left) : "-";
+        }
+    }
+}
+
+sub format_time {
+    my $time = shift;
+    my $s = "";
+
+    $s .= int($time/60/60)."hr "
+        if $time > 60*60;
+    $s .= int(($time % (60*60))/60)."min "
+        if $time > 60;
+    $s .= int($time % 60)."s"
+        if $time < 60*60;
+
+    return $s;
+}

commit 1c979c032b69cf4c578321eccb5719f71a7e01de
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 9 17:57:18 2011 -0500

    Move modules around to give a clear place for common methods

diff --git a/lib/RT/Importer.pm b/lib/RT/Migrate/Importer.pm
similarity index 99%
rename from lib/RT/Importer.pm
rename to lib/RT/Migrate/Importer.pm
index 6bde71a..c214f1f 100644
--- a/lib/RT/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -46,7 +46,7 @@
 #
 # END BPS TAGGED BLOCK }}}
 
-package RT::Importer;
+package RT::Migrate::Importer;
 
 use strict;
 use warnings;
diff --git a/lib/RT/Serializer.pm b/lib/RT/Migrate/Serializer.pm
similarity index 99%
rename from lib/RT/Serializer.pm
rename to lib/RT/Migrate/Serializer.pm
index a8d6981..0072e55 100644
--- a/lib/RT/Serializer.pm
+++ b/lib/RT/Migrate/Serializer.pm
@@ -46,7 +46,7 @@
 #
 # END BPS TAGGED BLOCK }}}
 
-package RT::Serializer;
+package RT::Migrate::Serializer;
 
 use strict;
 use warnings;
diff --git a/sbin/rt-importer b/sbin/rt-importer
index f2f43c8..8ac1607 100644
--- a/sbin/rt-importer
+++ b/sbin/rt-importer
@@ -7,7 +7,9 @@ use RT;
 RT::LoadConfig();
 RT::Init();
 
+use RT::Migrate::Importer;
 use Getopt::Long;
+use Pod::Usage qw//;
 use Time::HiRes qw//;
 
 my %OPT;
@@ -27,8 +29,7 @@ if ($summary) {
     $estimates{total} = $total;
 }
 
-use RT::Importer;
-my $import = RT::Importer->new(
+my $import = RT::Migrate::Importer->new(
     PreserveTicketIds => $OPT{'preserve-tickets'},
     OriginalId        => $OPT{originalid},
     Progress          => (%estimates ? \&progress : undef),
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index fcea548..e584a2c 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -7,6 +7,7 @@ use RT;
 RT::LoadConfig();
 RT::Init();
 
+use RT::Migrate::Serializer;
 use Getopt::Long;
 use Pod::Usage qw//;
 use Time::HiRes qw//;
@@ -58,8 +59,7 @@ if (-t STDOUT) {
     $args{Verbose}  = 0;
 }
 
-use RT::Serializer;
-my $walker = RT::Serializer->new( %args );
+my $walker = RT::Migrate::Serializer->new( %args );
 
 my %estimates = estimate();
 

commit 692e0bdbbcd8d2ede42d78e17888b02643ecb92a
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 9 19:23:07 2011 -0500

    Use RT::Migrate for display code shared between serializer and exporter

diff --git a/lib/RT/Migrate.pm b/lib/RT/Migrate.pm
new file mode 100644
index 0000000..089e4f1
--- /dev/null
+++ b/lib/RT/Migrate.pm
@@ -0,0 +1,171 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2011 Best Practical Solutions, LLC
+#                                          <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions,  LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+package RT::Migrate;
+
+use strict;
+use warnings;
+
+use Time::HiRes qw//;
+
+sub format_time {
+    my $time = shift;
+    my $s = "";
+
+    $s .= int($time/60/60)."hr "
+        if $time > 60*60;
+    $s .= int(($time % (60*60))/60)."min "
+        if $time > 60;
+    $s .= int($time % 60)."s"
+        if $time < 60*60;
+
+    return $s;
+}
+
+sub progress_bar {
+    my %args = (
+        label => "",
+        now   => 0,
+        max   => 1,
+        cols  => 80,
+        char  => "=",
+        @_,
+    );
+    $args{now} ||= 0;
+
+    my $fraction = $args{max} ? $args{now} / $args{max} : 0;
+
+    my $max_width = $args{cols} - 30;
+    my $bar_width = int($max_width * $fraction);
+
+    return sprintf "%20s |%-" . $max_width . "s|\n",
+        $args{label}, $args{char} x $bar_width;
+}
+
+sub progress {
+    my %args = (
+        top    => sub { print "\n\n" },
+        bottom => sub {},
+        every  => 3,
+        bars   => [qw/Ticket Transaction Attachment User Group/],
+        counts => sub {},
+        max    => {},
+        @_,
+    );
+
+    my $max_objects = 0;
+    $max_objects += $_ for values %{ $args{max} };
+
+    my $last_time;
+    my $start;
+    my $left;
+    return sub {
+        my $obj = shift;
+        my $now = Time::HiRes::time();
+        return if defined $last_time and $now - $last_time <= $args{every};
+
+        $start = $now unless $start;
+        $last_time = $now;
+
+        my $elapsed = $now - $start;
+
+        # Determine terminal size
+        print `clear`;
+        my ($cols, $rows) = (80, 25);
+        eval {
+            require Term::ReadKey;
+            ($cols, $rows) = Term::ReadKey::GetTerminalSize();
+        };
+        $cols -= 1;
+
+        $args{top}->($elapsed, $rows, $cols);
+
+        my %counts = $args{counts}->();
+        for my $class (map {"RT::$_"} @{$args{bars}}) {
+            my $display = $class;
+            $display =~ s/^RT::(.*)/@{[$1]}s:/;
+            print progress_bar(
+                label => $display,
+                now   => $counts{$class},
+                max   => $args{max}{$class},
+                cols  => $cols,
+            );
+        }
+
+        my $total = 0;
+        $total += $_ for values %counts;
+        print "\n", progress_bar(
+            label => "Total",
+            now   => $total,
+            max   => $max_objects,
+            cols  => $cols,
+            char  => "#",
+        );
+
+        # Time estimates
+        my $fraction = $max_objects ? $total/$max_objects : 0;
+        if ($fraction > 0.03) {
+            if (defined $left) {
+                $left = 0.75 * $left
+                      + 0.25 * ($elapsed / $fraction - $elapsed);
+            } else {
+                $left = ($elapsed / $fraction - $elapsed);
+            }
+        }
+        print "\n";
+        printf "%20s %s\n", "Elapsed time:",
+            format_time($elapsed);
+        printf "%20s %s\n", "Estimated left:",
+            (defined $left) ? format_time($left) : "-";
+
+        $args{bottom}->($elapsed, $rows, $cols);
+    }
+
+}
+
+1;
diff --git a/sbin/rt-importer b/sbin/rt-importer
index 8ac1607..8e716b1 100644
--- a/sbin/rt-importer
+++ b/sbin/rt-importer
@@ -7,6 +7,7 @@ use RT;
 RT::LoadConfig();
 RT::Init();
 
+use RT::Migrate;
 use RT::Migrate::Importer;
 use Getopt::Long;
 use Pod::Usage qw//;
@@ -19,20 +20,22 @@ GetOptions(
     "originalid|i=s",
 );
 
-my %estimates;
+my $import;
+
+my $progress,
 my ($summary) = grep {-f $_} map {"$_/rt-serialized"} @ARGV;
 if ($summary) {
     my $ref = Storable::retrieve($summary);
-    %estimates = %{ $ref->{counts} };
-    my $total = 0;
-    $total += $_ for values %estimates;
-    $estimates{total} = $total;
+    $progress = RT::Migrate::progress(
+        counts => sub { $import->ObjectCount },
+        max    => $ref->{counts},
+    );
 }
 
-my $import = RT::Migrate::Importer->new(
+$import = RT::Migrate::Importer->new(
     PreserveTicketIds => $OPT{'preserve-tickets'},
     OriginalId        => $OPT{originalid},
-    Progress          => (%estimates ? \&progress : undef),
+    Progress          => $progress,
 );
 my @files = map {-d $_ ? <$_/*.dat> : $_} @ARGV;
 my %counts = $import->Import( @files );
@@ -41,86 +44,3 @@ print "Total object counts:\n";
 for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
     printf "%8d %s\n", $counts{$_}, $_;
 }
-
-
-{
-    my $last_time;
-    my $start;
-    my $left;
-    sub progress {
-        my $obj = shift;
-        my $now = Time::HiRes::time();
-        return if defined $last_time and $now - $last_time <= 3;
-
-        $start = $now unless $start;
-        $last_time = $now;
-
-        my $elapsed = $now - $start;
-
-        print `clear`;
-        my ($cols, $rows) = (80, 25);
-        eval {
-            require Term::ReadKey;
-            ($cols, $rows) = Term::ReadKey::GetTerminalSize();
-        };
-        $cols -= 1;
-
-        my %counts = $import->ObjectCount;
-        # Just show the big ones
-        for my $class (map {"RT::$_"} qw/Ticket Transaction Attachment User Group/) {
-            my $display = $class;
-            $display =~ s/^RT::(.*)/@{[$1]}s:/;
-
-            $counts{$class} ||= 0;
-            my $fraction = $counts{$class}/$estimates{$class};
-
-            my $max_width = $cols - 30;
-            my $bar_width = int($max_width * $fraction);
-
-            printf "%20s |%-".$max_width."s|\n",
-                $display, "=" x $bar_width;
-        }
-
-        {
-            my $total = 0;
-            $total += $_ for values %counts;
-            print "\n";
-
-            my $fraction = $total/$estimates{total};
-
-            my $max_width = $cols - 30;
-            my $bar_width = int($max_width * $fraction);
-
-            printf "%20s |%-".$max_width."s|\n",
-                "Total:", "#" x $bar_width;
-
-            if ($fraction > 0.03) {
-                if (defined $left) {
-                    $left = 0.75 * $left
-                          + 0.25 * ($elapsed / $fraction - $elapsed);
-                } else {
-                    $left = ($elapsed / $fraction - $elapsed);
-                }
-            }
-            print "\n";
-            printf "%20s %s\n", "Elapsed time:",
-                format_time($elapsed);
-            printf "%20s %s\n", "Estimated left:",
-                (defined $left) ? format_time($left) : "-";
-        }
-    }
-}
-
-sub format_time {
-    my $time = shift;
-    my $s = "";
-
-    $s .= int($time/60/60)."hr "
-        if $time > 60*60;
-    $s .= int(($time % (60*60))/60)."min "
-        if $time > 60;
-    $s .= int($time % 60)."s"
-        if $time < 60*60;
-
-    return $s;
-}
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index e584a2c..6e9b1c2 100644
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -7,6 +7,7 @@ use RT;
 RT::LoadConfig();
 RT::Init();
 
+use RT::Migrate;
 use RT::Migrate::Serializer;
 use Getopt::Long;
 use Pod::Usage qw//;
@@ -49,19 +50,24 @@ $args{FollowACL}     = $OPT{acls}     if defined $OPT{acls};
 
 $args{GC} = defined $OPT{gc} ? $OPT{gc} : 5000;
 
+my $walker;
+
 my $gnuplot = `which gnuplot`;
 my $msg = "";
 if (-t STDOUT) {
-    $args{Progress} = \&progress;
+    $args{Progress} = RT::Migrate::progress(
+        top    => \&gnuplot,
+        bottom => sub { print "\n$msg"; $msg = ""; },
+        counts => sub { $walker->ObjectCount },
+        max    => { estimate() },
+    );
     $args{MessageHandler} = sub {
         print "\r", " "x60, "\r", $_[-1]; $msg = $_[-1];
     };
     $args{Verbose}  = 0;
 }
 
-my $walker = RT::Migrate::Serializer->new( %args );
-
-my %estimates = estimate();
+$walker = RT::Migrate::Serializer->new( %args );
 
 print "Beginning database serialization...";
 my %counts = $walker->Walk;
@@ -83,26 +89,13 @@ sub estimate {
     my %e;
 
     # Expected types we'll serialize
-    my @types = qw/
-                      RT::Queue
-                      RT::Ticket
-                      RT::Transaction
-                      RT::Attachment
-                      RT::Link
-
-                      RT::User
-                      RT::Group
-                      RT::GroupMember
-
-                      RT::Attribute
-
-                      RT::CustomField
-                      RT::CustomFieldValue
-                      RT::ObjectCustomField
-                      RT::ObjectCustomFieldValue
-                  /;
-
-    $e{total} = 0;
+    my @types = map {"RT::$_"} qw/
+        Queue Ticket Transaction Attachment Link
+        User  Group  GroupMember Attribute
+        CustomField CustomFieldValue
+        ObjectCustomField ObjectCustomFieldValue
+                                 /;
+
     for my $class (@types) {
         print "Estimating $class count...";
         my $collection = $class . "s";
@@ -110,7 +103,6 @@ sub estimate {
         $objs->UnLimit;
         $objs->{allow_deleted_search} = 1 if $class eq "RT::Ticket";
         $e{$class} = $objs->Count;
-        $e{total} += $e{$class};
         print "\r", " "x60, "\r";
     }
 
@@ -118,118 +110,34 @@ sub estimate {
 }
 
 
-{
-    my $last_time;
-    my $start;
-    my $left;
-    sub progress {
-        my $obj = shift;
-        my $now = Time::HiRes::time();
-        return if defined $last_time and $now - $last_time <= 3;
-
-        $start = $now unless $start;
-        $last_time = $now;
-
-        my $elapsed = $now - $start;
-
-        print `clear`;
-        my ($cols, $rows) = (80, 25);
-        eval {
-            require Term::ReadKey;
-            ($cols, $rows) = Term::ReadKey::GetTerminalSize();
-        };
-        $cols -= 1;
-
-        if ($gnuplot) {
-            my $length = $walker->StackSize;
-            my $file = $walker->Directory . "/progress.plot";
-            open(my $dat, ">>", $file);
-            printf $dat "%10.3f\t%8d\n",
-                $elapsed,
-                $length;
-            close $dat;
-
-            if ($rows <= 24) {
-                print "\n\n";
-            } elsif ($elapsed) {
-                system("gnuplot", "-e", <<EOT );
-set term dumb $cols @{[$rows - 15]};
-set xlabel "Seconds";
-unset key;
-set autoscale;
-set title "Queue length";
-plot "$file" using 1:2 with lines
-EOT
-            } else {
-                print "\n" for 1..($rows - 14);
-            }
-        } else {
-            print "\n\n";
-        }
-
-        my %counts = $walker->ObjectCount;
-        # Just show the big ones
-        for my $class (map {"RT::$_"} qw/Ticket Transaction Attachment User Group/) {
-            my $display = $class;
-            $display =~ s/^RT::(.*)/@{[$1]}s:/;
-
-            $counts{$class} ||= 0;
-            my $fraction = $counts{$class}/$estimates{$class};
-
-            my $max_width = $cols - 30;
-            my $bar_width = int($max_width * $fraction);
-
-            printf "%20s |%-".$max_width."s|\n",
-                $display, "=" x $bar_width;
-        }
-
-        {
-            my $total = 0;
-            $total += $_ for values %counts;
-            print "\n";
-
-            my $fraction = $total/$estimates{total};
-
-            my $max_width = $cols - 30;
-            my $bar_width = int($max_width * $fraction);
-
-            printf "%20s |%-".$max_width."s|\n",
-                "Total:", "#" x $bar_width;
-
-            if ($fraction > 0.03) {
-                if (defined $left) {
-                    $left = 0.75 * $left
-                          + 0.25 * ($elapsed / $fraction - $elapsed);
-                } else {
-                    $left = ($elapsed / $fraction - $elapsed);
-                }
-            }
-            print "\n";
-            printf "%20s %s\n", "Elapsed time:",
-                format_time($elapsed);
-            printf "%20s %s\n", "Estimated left:",
-                (defined $left) ? format_time($left) : "-";
-        }
-
-        print "\n$msg";
-        $msg = "";
+sub gnuplot {
+    my ($elapsed, $rows, $cols) = @_;
+    my $length = $walker->StackSize;
+    my $file = $walker->Directory . "/progress.plot";
+    open(my $dat, ">>", $file);
+    printf $dat "%10.3f\t%8d\n", $elapsed, $length;
+    close $dat;
+
+    if ($rows <= 24 or not $gnuplot) {
+        print "\n\n";
+    } elsif ($elapsed) {
+        my $gnuplot = qx|
+            gnuplot -e '
+                set term dumb $cols @{[$rows - 12]};
+                set xlabel "Seconds";
+                unset key;
+                set autoscale;
+                set title "Queue length";
+                plot "$file" using 1:2 with lines
+            '
+        |;
+        $gnuplot =~ s/^(\s*\n)//;
+        print $gnuplot;
+    } else {
+        print "\n" for 1..($rows - 13);
     }
 }
 
-sub format_time {
-    my $time = shift;
-    my $s = "";
-
-    $s .= int($time/60/60)."hr "
-        if $time > 60*60;
-    $s .= int(($time % (60*60))/60)."min "
-        if $time > 60;
-    $s .= int($time % 60)."s"
-        if $time < 60*60;
-
-    return $s;
-}
-
 =head1 NAME
 
 rt-serializer - Serialize an RT database to disk

commit c12d742295646c611bf6d7fd8a2e3b9f371c4c09
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 21 16:25:39 2011 -0500

    Catch lack of UID during dependency insertion, where we have the referrer

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index acf9724..299a335 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -240,8 +240,11 @@ sub AppendDeps {
     my $self = shift;
     my ($dir, $deps, $from) = @_;
     for my $obj (@{$deps->{$dir}}) {
-        if ($obj->isa("RT::Record")) {
-            next unless $obj->id;
+        if (not defined $obj) {
+            warn "$dir from $from contained an invalid reference";
+            next;
+        } elsif ($obj->isa("RT::Record")) {
+            warn "$dir from $from to $obj is an invalid reference" unless $obj->UID;
             next if $self->{GC} < 0 and exists $self->{seen}{$obj->UID};
         } else {
             $obj->FindAllRows;
@@ -262,8 +265,11 @@ sub PrependDeps {
     my $self = shift;
     my ($dir, $deps, $from) = @_;
     for my $obj (@{$deps->{$dir}}) {
-        if ($obj->isa("RT::Record")) {
-            next unless $obj->id;
+        if (not defined $obj) {
+            warn "$dir from $from contained an invalid reference";
+            next;
+        } elsif ($obj->isa("RT::Record")) {
+            warn "$dir from $from to $obj is an invalid reference" unless $obj->UID;
             next if $self->{GC} < 0 and exists $self->{visited}{$obj->UID};
         } else {
             $obj->FindAllRows;
diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 6e5dca2..863fff1 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -1939,7 +1939,7 @@ sub WikiBase {
 
 sub UID {
     my $self = shift;
-    warn Carp::longmess("No ID for $self") unless defined $self->Id;
+    return undef unless defined $self->Id;
     return "@{[ref $self]}-$RT::Organization-@{[$self->Id]}";
 }
 
diff --git a/lib/RT/User.pm b/lib/RT/User.pm
index 37413a6..b080bc2 100644
--- a/lib/RT/User.pm
+++ b/lib/RT/User.pm
@@ -2286,7 +2286,7 @@ sub _CoreAccessible {
 
 sub UID {
     my $self = shift;
-    warn Carp::longmess("No Name for $self") unless defined $self->Name;
+    return undef unless defined $self->Name;
     return "@{[ref $self]}-@{[$self->Name]}";
 }
 

commit e47e72f829e3557035ffefb5c497e1025f596e14
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 21 16:25:55 2011 -0500

    chmod +x new sbin/ files

diff --git a/sbin/rt-importer b/sbin/rt-importer
old mode 100644
new mode 100755
diff --git a/sbin/rt-serializer b/sbin/rt-serializer
old mode 100644
new mode 100755

commit 7eb38edb3fa46245ebfc8da785c9e7555e014dc8
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 21 16:28:17 2011 -0500

    Add a short form of --force

diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index 6e9b1c2..e72678c 100755
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -19,7 +19,7 @@ GetOptions(
     "help|?",
 
     "directory|d=s",
-    "force!",
+    "force|f!",
     "size|s=i",
 
     "users!",

commit d1bb86fb60ad2dd839370f27e192bc284a551ba7
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 21 16:28:51 2011 -0500

    Add --quiet and --verbose for non-ASCII-art modes

diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index e72678c..cdff944 100755
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -17,6 +17,8 @@ my %OPT;
 GetOptions(
     \%OPT,
     "help|?",
+    "verbose|v!",
+    "quiet|q!",
 
     "directory|d=s",
     "force|f!",
@@ -54,7 +56,7 @@ my $walker;
 
 my $gnuplot = `which gnuplot`;
 my $msg = "";
-if (-t STDOUT) {
+if (-t STDOUT and not $OPT{verbose} and not $OPT{quiet}) {
     $args{Progress} = RT::Migrate::progress(
         top    => \&gnuplot,
         bottom => sub { print "\n$msg"; $msg = ""; },
@@ -66,6 +68,8 @@ if (-t STDOUT) {
     };
     $args{Verbose}  = 0;
 }
+$args{Verbose} = 0 if $OPT{quiet};
+
 
 $walker = RT::Migrate::Serializer->new( %args );
 
@@ -217,6 +221,15 @@ Skip serialization of all ticket data.
 Adjust how often the garbage collection sweep is done; lower numbers are
 more frequent.  See L</GARBAGE COLLECTION>.
 
+=item B<--quiet>
+
+Do not show graphical progress UI.
+
+=item B<--verbose>
+
+Do not show graphical progress UI, but rather log was each row is
+written out.
+
 =back
 
 =head1 GARBAGE COLLECTION

commit fe178ad5ff48ad28dc7846cb3ae88bc620f09308
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 21 16:29:12 2011 -0500

    Make the default message handler be more useful

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index 299a335..48f911e 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -52,6 +52,7 @@ use strict;
 use warnings;
 
 use RT::DependencyWalker::Dependencies;
+use Carp;
 
 sub new {
     my $class = shift;
@@ -63,10 +64,10 @@ sub new {
 sub Init {
     my $self = shift;
     my %args = (
-        First         => "top",
-        GC            => 0,
-        Progress      => undef,
-        MessageHandler => sub { warn for @_ },
+        First          => "top",
+        GC             => 0,
+        Progress       => undef,
+        MessageHandler => \&Carp::carp,
         @_
     );
 

commit 7695c3ef274b9cc81cd58d9fe70c7d3aa94451ac
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 21 16:52:43 2011 -0500

    Rename ->Dependencies to ->FindDependencies to not conflict with Shredder

diff --git a/lib/RT/ACE.pm b/lib/RT/ACE.pm
index 974150b..833ebfe 100644
--- a/lib/RT/ACE.pm
+++ b/lib/RT/ACE.pm
@@ -744,11 +744,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->PrincipalObj->Object );
     $deps->Add( out => $self->Object );
diff --git a/lib/RT/Article.pm b/lib/RT/Article.pm
index 33e46b7..6d3bfce 100644
--- a/lib/RT/Article.pm
+++ b/lib/RT/Article.pm
@@ -862,11 +862,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
     $deps->Add( out => $self->ClassObj );
     $deps->Add( in => $self->Topics );
 }
diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index 95c2ebf..be57ce4 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -1035,11 +1035,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
     $deps->Add( out => $self->TransactionObj );
 }
 
diff --git a/lib/RT/Attribute.pm b/lib/RT/Attribute.pm
index f231186..bb3b93d 100644
--- a/lib/RT/Attribute.pm
+++ b/lib/RT/Attribute.pm
@@ -627,11 +627,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
     $deps->Add( out => $self->Object );
 }
 
diff --git a/lib/RT/Class.pm b/lib/RT/Class.pm
index 9043024..2e6eee2 100644
--- a/lib/RT/Class.pm
+++ b/lib/RT/Class.pm
@@ -616,11 +616,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     my $articles = RT::Articles->new( $self->CurrentUser );
     $articles->Limit( FIELD => "Class", VALUE => $self->Id );
diff --git a/lib/RT/CustomField.pm b/lib/RT/CustomField.pm
index 2127093..ec4b1f8 100644
--- a/lib/RT/CustomField.pm
+++ b/lib/RT/CustomField.pm
@@ -1988,11 +1988,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->BasedOnObj )
         if $self->BasedOnObj->id;
diff --git a/lib/RT/CustomFieldValue.pm b/lib/RT/CustomFieldValue.pm
index f733e55..a92638a 100644
--- a/lib/RT/CustomFieldValue.pm
+++ b/lib/RT/CustomFieldValue.pm
@@ -329,11 +329,11 @@ sub _CoreAccessible {
 };
 
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->CustomFieldObj );
 }
diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index 48f911e..35a1a6d 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -203,8 +203,8 @@ sub Process {
         # stack, then objects it refers to.
         return if defined $args{from}
             and not $self->Observe(%args);
-        my $deps = RT::DependencyWalker::Dependencies->new;
-        $obj->Dependencies($self, $deps);
+        my $deps = RT::DependencyWalker::FindDependencies->new;
+        $obj->FindDependencies($self, $deps);
         # Shove it back for later
         push @{$self->{replace}}, \%args;
         if ($self->{first} eq "top") {
diff --git a/lib/RT/DependencyWalker/Dependencies.pm b/lib/RT/DependencyWalker/Dependencies.pm
index b17e4ab..1f842c8 100644
--- a/lib/RT/DependencyWalker/Dependencies.pm
+++ b/lib/RT/DependencyWalker/Dependencies.pm
@@ -46,7 +46,7 @@
 #
 # END BPS TAGGED BLOCK }}}
 
-package RT::DependencyWalker::Dependencies;
+package RT::DependencyWalker::FindDependencies;
 
 use strict;
 use warnings;
diff --git a/lib/RT/Group.pm b/lib/RT/Group.pm
index e756696..1987f07 100644
--- a/lib/RT/Group.pm
+++ b/lib/RT/Group.pm
@@ -1413,11 +1413,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     my $instance = $self->InstanceObj;
     $deps->Add( out => $instance ) if $instance;
diff --git a/lib/RT/GroupMember.pm b/lib/RT/GroupMember.pm
index 27dbf68..f7311e2 100644
--- a/lib/RT/GroupMember.pm
+++ b/lib/RT/GroupMember.pm
@@ -496,11 +496,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->GroupObj->Object );
     $deps->Add( out => $self->MemberObj->Object );
diff --git a/lib/RT/Link.pm b/lib/RT/Link.pm
index 02e99dc..e88892d 100644
--- a/lib/RT/Link.pm
+++ b/lib/RT/Link.pm
@@ -446,11 +446,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->BaseObj )   if $self->BaseObj   and $self->BaseObj->id;
     $deps->Add( out => $self->TargetObj ) if $self->TargetObj and $self->TargetObj->id;
diff --git a/lib/RT/ObjectClass.pm b/lib/RT/ObjectClass.pm
index 5901406..2b9bc62 100644
--- a/lib/RT/ObjectClass.pm
+++ b/lib/RT/ObjectClass.pm
@@ -216,11 +216,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->ClassObj );
 
diff --git a/lib/RT/ObjectCustomField.pm b/lib/RT/ObjectCustomField.pm
index 40a1229..444c7d3 100644
--- a/lib/RT/ObjectCustomField.pm
+++ b/lib/RT/ObjectCustomField.pm
@@ -407,11 +407,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->CustomFieldObj );
 
diff --git a/lib/RT/ObjectCustomFieldValue.pm b/lib/RT/ObjectCustomFieldValue.pm
index 208af5e..7b181df 100644
--- a/lib/RT/ObjectCustomFieldValue.pm
+++ b/lib/RT/ObjectCustomFieldValue.pm
@@ -751,11 +751,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->CustomFieldObj );
     $deps->Add( out => $self->Object );
diff --git a/lib/RT/ObjectTopic.pm b/lib/RT/ObjectTopic.pm
index f7b2c19..b986acc 100644
--- a/lib/RT/ObjectTopic.pm
+++ b/lib/RT/ObjectTopic.pm
@@ -209,11 +209,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->TopicObj );
 
diff --git a/lib/RT/Queue.pm b/lib/RT/Queue.pm
index 140dbcc..c5b4b1c 100644
--- a/lib/RT/Queue.pm
+++ b/lib/RT/Queue.pm
@@ -1537,11 +1537,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     # Queue role groups( Cc, AdminCc )
     my $objs = RT::Groups->new( $self->CurrentUser );
diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 863fff1..c79b418 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -1943,7 +1943,7 @@ sub UID {
     return "@{[ref $self]}-$RT::Organization-@{[$self->Id]}";
 }
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
     for my $col (qw/Creator LastUpdatedBy/) {
diff --git a/lib/RT/Scrip.pm b/lib/RT/Scrip.pm
index 2fd6a2f..0790c30 100644
--- a/lib/RT/Scrip.pm
+++ b/lib/RT/Scrip.pm
@@ -994,11 +994,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->ScripConditionObj );
     $deps->Add( out => $self->ScripActionObj );
diff --git a/lib/RT/Template.pm b/lib/RT/Template.pm
index f90369e..68fcc08 100644
--- a/lib/RT/Template.pm
+++ b/lib/RT/Template.pm
@@ -943,11 +943,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->QueueObj );
 }
diff --git a/lib/RT/Ticket.pm b/lib/RT/Ticket.pm
index fc72267..4849ea6 100644
--- a/lib/RT/Ticket.pm
+++ b/lib/RT/Ticket.pm
@@ -4175,11 +4175,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     # Links
     my $links = RT::Links->new( $self->CurrentUser );
diff --git a/lib/RT/Topic.pm b/lib/RT/Topic.pm
index 6ce74a4..0440f35 100644
--- a/lib/RT/Topic.pm
+++ b/lib/RT/Topic.pm
@@ -372,11 +372,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
     $deps->Add( out => $self->ParentObj );
     $deps->Add( in => $self->Children );
 
diff --git a/lib/RT/Transaction.pm b/lib/RT/Transaction.pm
index d2f885c..ea74946 100644
--- a/lib/RT/Transaction.pm
+++ b/lib/RT/Transaction.pm
@@ -1608,11 +1608,11 @@ sub _CoreAccessible {
  }
 };
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     $deps->Add( out => $self->Object );
     $deps->Add( in => $self->Attachments );
diff --git a/lib/RT/User.pm b/lib/RT/User.pm
index b080bc2..d60c953 100644
--- a/lib/RT/User.pm
+++ b/lib/RT/User.pm
@@ -2290,11 +2290,11 @@ sub UID {
     return "@{[ref $self]}-@{[$self->Name]}";
 }
 
-sub Dependencies {
+sub FindDependencies {
     my $self = shift;
     my ($walker, $deps) = @_;
 
-    $self->SUPER::Dependencies($walker, $deps);
+    $self->SUPER::FindDependencies($walker, $deps);
 
     # ACL equivalence group
     my $objs = RT::Groups->new( $self->CurrentUser );

commit f649a291105a48bc95a8f55efbebe777944a8df3
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 23 21:39:58 2011 -0500

    Safeguard if ->Process gets an invalid object

diff --git a/lib/RT/DependencyWalker.pm b/lib/RT/DependencyWalker.pm
index 35a1a6d..6c3f54d 100644
--- a/lib/RT/DependencyWalker.pm
+++ b/lib/RT/DependencyWalker.pm
@@ -182,6 +182,10 @@ sub Process {
     return if $obj->isa("RT::System");
 
     my $uid = $obj->UID;
+    unless ($uid) {
+        warn "$args{direction} from $args{from} to $obj is an invalid reference";
+        return;
+    }
     $self->{progress}->($obj) if $self->{progress};
     if (exists $self->{visited}{$uid}) {
         # Already visited -- no-op

commit 22b0d7f2ef6edcff5ae5175ed80a47f3d930e933
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 23 21:40:18 2011 -0500

    Catch and warn about missing system users and groups

diff --git a/lib/RT/Migrate/Serializer.pm b/lib/RT/Migrate/Serializer.pm
index 0072e55..52c5b8c 100644
--- a/lib/RT/Migrate/Serializer.pm
+++ b/lib/RT/Migrate/Serializer.pm
@@ -119,15 +119,17 @@ sub PushBasics {
     # System users
     for my $name (qw/RT_System root nobody/) {
         my $user = RT::User->new( RT->SystemUser );
-        $user->Load( $name );
-        $self->PushObj( $user );
+        my ($id, $msg) = $user->Load( $name );
+        warn "No '$name' user found: $msg" unless $id;
+        $self->PushObj( $user ) if $id;
     }
 
     # System groups
     foreach my $name (qw(Everyone Privileged Unprivileged)) {
         my $group = RT::Group->new( RT->SystemUser );
-        $group->LoadSystemInternalGroup( $name );
-        $self->PushObj( $group );
+        my ($id, $msg) = $group->LoadSystemInternalGroup( $name );
+        warn "No '$name' group found: $msg" unless $id;
+        $self->PushObj( $group ) if $id;
     }
 
     # System role groups

commit 1ed50ccde11b00696162b30d52f502d869a833d2
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 28 11:19:58 2011 -0500

    Give RT::Topic an Object method to go with its ObjectId  and ObjectType

diff --git a/lib/RT/Topic.pm b/lib/RT/Topic.pm
index 0440f35..ea53b64 100644
--- a/lib/RT/Topic.pm
+++ b/lib/RT/Topic.pm
@@ -226,11 +226,7 @@ sub CurrentUserHasRight {
     my $right = shift;
 
     my $equiv = [ $RT::System ];
-    if ($self->ObjectId) {
-        my $obj = $self->ObjectType->new($self->CurrentUser);
-        $obj->Load($self->ObjectId);
-        push @{$equiv}, $obj;
-    }
+    push @{$equiv}, $self->Object if $self->ObjectId;
     if ($self->Id) {
         return ( $self->CurrentUser->HasRight(
                                               Right        => $right,
@@ -253,6 +249,13 @@ sub CurrentUserHasRight {
 # }}}
 
 
+sub Object {
+    my $self  = shift;
+    my $Object = $self->__Value('ObjectType')->new( $self->CurrentUser );
+    $Object->Load( $self->__Value('ObjectId') );
+    return $Object;
+}
+
 =head2 id
 
 Returns the current value of id. 
@@ -378,11 +381,8 @@ sub FindDependencies {
 
     $self->SUPER::FindDependencies($walker, $deps);
     $deps->Add( out => $self->ParentObj );
-    $deps->Add( in => $self->Children );
-
-    my $obj = $self->ObjectType->new( $self->CurrentUser );
-    $obj->Load( $self->ObjectId );
-    $deps->Add( out => $obj );
+    $deps->Add( in  => $self->Children );
+    $deps->Add( out => $self->Object );
 }
 
 RT::Base->_ImportOverlays();

commit ebcd9671b39084ddfa36f125f78d5b86a22d4d21
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 28 11:21:49 2011 -0500

    Error-proof by checking that there is an Object method to serialize

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index c79b418..f0b43c1 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -2030,7 +2030,7 @@ sub Serialize {
     delete $store{id};
 
     # Anything on an object should get the UID stored instead
-    if ($store{ObjectType} and $store{ObjectId}) {
+    if ($store{ObjectType} and $store{ObjectId} and $self->can("Object")) {
         delete $store{$_} for qw/ObjectType ObjectId/;
         $store{Object} = \($self->Object->UID);
     }

commit 56120cac357c3c607d2807977ee6fc8e3aa72981
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 28 11:46:04 2011 -0500

    Only chase the ParentObj if it exists

diff --git a/lib/RT/Topic.pm b/lib/RT/Topic.pm
index ea53b64..3db8a64 100644
--- a/lib/RT/Topic.pm
+++ b/lib/RT/Topic.pm
@@ -380,7 +380,7 @@ sub FindDependencies {
     my ($walker, $deps) = @_;
 
     $self->SUPER::FindDependencies($walker, $deps);
-    $deps->Add( out => $self->ParentObj );
+    $deps->Add( out => $self->ParentObj ) if $self->ParentObj->Id;
     $deps->Add( in  => $self->Children );
     $deps->Add( out => $self->Object );
 }

commit f98bcc47470821d0bc07d5c80bdd1d9694a7c04d
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Nov 28 12:51:17 2011 -0500

    Add classes from RTFM to InitClasses' list

diff --git a/lib/RT.pm b/lib/RT.pm
index f1e27cc..d62ba24 100644
--- a/lib/RT.pm
+++ b/lib/RT.pm
@@ -449,6 +449,10 @@ sub InitClasses {
         RT::ObjectCustomFieldValue
         RT::Attribute
         RT::Link
+        RT::Topic
+        RT::Class
+        RT::Article
+        RT::ObjectTopic
     );
 
     if ( $args{'Heavy'} ) {

commit 5774e00d13dd228ead008f6647679789044a3a6b
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 29 03:48:57 2011 -0500

    Tack the necessary package name onto the call to Dumper

diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index c214f1f..0539161 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -246,7 +246,8 @@ sub Import {
             );
             unless ($id) {
                 require Data::Dumper;
-                warn "Failed to create $uid: $msg\n" . Dumper($data);
+                warn "Failed to create $uid: $msg\n"
+                    . Data::Dumper::Dumper($data);
                 next;
             }
 

commit d56165edfdde9010d7e947108edb7b5e64df7d0e
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 29 23:03:32 2011 -0500

    Factor out object creation from the loop

diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index 0539161..4c9f1ab 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -204,6 +204,33 @@ sub MergeBy {
     return 1;
 }
 
+sub Create {
+    my $self = shift;
+    my ($class, $uid, $data) = @_;
+    return unless $class->PreInflate( $self, $uid, $data );
+
+    # Remove the ticket id, unless we specifically want it kept
+    delete $data->{id} if $class eq "RT::Ticket"
+        and not $self->{PreserveTicketIds};
+
+    my $obj = $class->new( RT->SystemUser );
+    my ($id, $msg) = $obj->DBIx::SearchBuilder::Record::Create(
+        %{$data}
+    );
+    die "Failed to create $uid: $msg\n" . Data::Dumper::Dumper($data) . "\n"
+        unless $id;
+
+    $self->{ObjectCount}{$class}++;
+    $self->Resolve( $uid => $class, $id );
+
+    # Load it back to get real values into the columns
+    $obj = $class->new( RT->SystemUser );
+    $obj->Load( $id );
+    $obj->PostInflate( $self );
+
+    return $obj;
+}
+
 sub Import {
     my $self = shift;
     my @files = @_;
@@ -234,36 +261,14 @@ sub Import {
             push @{$new{$class}}, $uid
                 if $class eq "RT::Queue";
 
-            next unless $class->PreInflate( $self, $uid, $data );
-
-            # Remove the ticket id, unless we specifically want it kept
-            delete $data->{id} if $class eq "RT::Ticket"
-                and not $self->{PreserveTicketIds};
-
-            my $obj = $class->new( RT->SystemUser );
-            my ($id, $msg) = $obj->DBIx::SearchBuilder::Record::Create(
-                %{$data}
-            );
-            unless ($id) {
-                require Data::Dumper;
-                warn "Failed to create $uid: $msg\n"
-                    . Data::Dumper::Dumper($data);
-                next;
-            }
-
-            $self->{ObjectCount}{$class}++;
-            $self->Resolve( $uid => $class, $id );
-
-            # Load it back to get real values into the columns
-            $obj = $class->new( RT->SystemUser );
-            $obj->Load( $id );
-            $obj->PostInflate( $self );
+            my $obj = $self->Create( $class, $uid, $data );
+            next unless $obj;
 
             # If it's a ticket, we might need to create a
             # TicketCustomField for the previous ID
             if ($class eq "RT::Ticket" and $self->{OriginalId}) {
                 my ($org, $origid) = $uid =~ /^RT::Ticket-(.*)-(\d+)$/;
-                ($id, $msg) = $obj->AddCustomFieldValue(
+                my ($id, $msg) = $obj->AddCustomFieldValue(
                     Field             => $self->{OriginalId},
                     Value             => "$org:$origid",
                     RecordTransaction => 0,

commit cc7149054eb7449a890ef3d4dc9d681d26c8226d
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 29 23:05:15 2011 -0500

    Move more prep into ->new

diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index 4c9f1ab..6825d94 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -103,6 +103,10 @@ sub Init {
 
     # What we created
     $self->{ObjectCount} = {};
+
+    # Basic facts of life, as a safety net
+    $self->Resolve( RT->System->UID => ref RT->System, RT->System->Id );
+    $self->SkipTransactions( RT->System->UID );
 }
 
 sub Resolve {
@@ -243,8 +247,6 @@ sub Import {
         return $self->Id;
     };
 
-    $self->Resolve( RT->System->UID => ref RT->System, RT->System->Id );
-    $self->SkipTransactions( RT->System->UID );
 
     my %unglobal;
     my %new;

commit b69283621888f81d5c9bfe1fb58c2891ee117875
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 29 23:06:41 2011 -0500

    Move unglobal'ing into object variables, as it is state we might need to serialize

diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index 6825d94..c326767 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -104,6 +104,10 @@ sub Init {
     # What we created
     $self->{ObjectCount} = {};
 
+    # To know what global CFs need to be unglobal'd and applied to what
+    $self->{NewQueues} = [];
+    $self->{NewCFs} = [];
+
     # Basic facts of life, as a safety net
     $self->Resolve( RT->System->UID => ref RT->System, RT->System->Id );
     $self->SkipTransactions( RT->System->UID );
@@ -247,9 +251,6 @@ sub Import {
         return $self->Id;
     };
 
-
-    my %unglobal;
-    my %new;
     for my $f (@files) {
         open(my $fh, "<", $f) or die "Can't read $f: $!";
         while (not eof($fh)) {
@@ -260,7 +261,7 @@ sub Import {
             # it to split global CFs into non-global across those
             # fields.  We do this before inflating, so that queues which
             # got merged still get the CFs applied
-            push @{$new{$class}}, $uid
+            push @{$self->{NewQueues}}, $uid
                 if $class eq "RT::Queue";
 
             my $obj = $self->Create( $class, $uid, $data );
@@ -282,7 +283,7 @@ sub Import {
             # If it's a CF, we don't know yet if it's global (the OCF
             # hasn't been created yet) to store away the CF for later
             # inspection
-            push @{$unglobal{"RT::Queue"}}, $uid
+            push @{$self->{NewCFs}}, $uid
                 if $class eq "RT::CustomField"
                     and $obj->LookupType =~ /^RT::Queue/;
 
@@ -291,16 +292,14 @@ sub Import {
     }
 
     # Take global CFs which we made and make them un-global
-    for my $class (keys %unglobal) {
-        my @objs = grep {$_} map {$self->LookupObj( $_ )} @{$new{$class}};
-
-        for my $uid (@{$unglobal{$class}}) {
-            my $obj = $self->LookupObj( $uid );
-            my $ocf = $obj->IsApplied( 0 ) or next;
-            $ocf->Delete;
-            $obj->AddToObject( $_ ) for @objs;
-        }
+    my @queues = grep {$_} map {$self->LookupObj( $_ )} @{$self->{NewQueues}};
+    for my $obj (map {$self->LookupObj( $_ )} @{$self->{NewCFs}}) {
+        my $ocf = $obj->IsApplied( 0 ) or next;
+        $ocf->Delete;
+        $obj->AddToObject( $_ ) for @queues;
     }
+    $self->{NewQueues} = [];
+    $self->{NewCFs} = [];
 
     # Anything we didn't see is an error
     if (keys %{$self->{Pending}}) {

commit 726c268a40bb4d231fc7edcb1e0ce1cb06e157b3
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 29 23:07:11 2011 -0500

    Standardize on one case for internal properties

diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index c326767..fdcabd8 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -169,13 +169,13 @@ sub Postpone {
 sub SkipTransactions {
     my $self = shift;
     my ($uid) = @_;
-    $self->{skiptransactions}{$uid} = 1;
+    $self->{SkipTransactions}{$uid} = 1;
 }
 
 sub ShouldSkipTransaction {
     my $self = shift;
     my ($uid) = @_;
-    return exists $self->{skiptransactions}{$uid};
+    return exists $self->{SkipTransactions}{$uid};
 }
 
 sub MergeValues {

commit 2df3fc2185967002e1985985478dc8f7906111b2
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 29 23:11:54 2011 -0500

    Import now just takes a directory -- it's much more straightforard that way

diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index fdcabd8..3a55750 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -241,7 +241,9 @@ sub Create {
 
 sub Import {
     my $self = shift;
-    my @files = @_;
+    my ($dir) = @_;
+
+    my @files = <$dir/*.dat>;
 
     no warnings 'redefine';
     local *RT::Ticket::Load = sub {
diff --git a/sbin/rt-importer b/sbin/rt-importer
index 8e716b1..9e96479 100755
--- a/sbin/rt-importer
+++ b/sbin/rt-importer
@@ -20,12 +20,15 @@ GetOptions(
     "originalid|i=s",
 );
 
+die "Usage: rt-importer directory/\n" unless @ARGV == 1;
+my ($dir) = @ARGV;
+$dir =~ s|/$||;
+
 my $import;
 
-my $progress,
-my ($summary) = grep {-f $_} map {"$_/rt-serialized"} @ARGV;
-if ($summary) {
-    my $ref = Storable::retrieve($summary);
+my $progress;
+if (-f "$dir/rt-serialized") {
+    my $ref = Storable::retrieve("$dir/rt-serialized");
     $progress = RT::Migrate::progress(
         counts => sub { $import->ObjectCount },
         max    => $ref->{counts},
@@ -37,9 +40,8 @@ $import = RT::Migrate::Importer->new(
     OriginalId        => $OPT{originalid},
     Progress          => $progress,
 );
-my @files = map {-d $_ ? <$_/*.dat> : $_} @ARGV;
-my %counts = $import->Import( @files );
 
+my %counts = $import->Import( $dir );
 print "Total object counts:\n";
 for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
     printf "%8d %s\n", $counts{$_}, $_;

commit 48ce8bf29557b441cf48c19231f7d364305de2f7
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 29 23:22:54 2011 -0500

    Store internally current file and position, as well as remaining files

diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index 3a55750..cd456ca 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -52,6 +52,7 @@ use strict;
 use warnings;
 
 use Storable qw//;
+use File::Spec;
 
 sub new {
     my $class = shift;
@@ -243,7 +244,7 @@ sub Import {
     my $self = shift;
     my ($dir) = @_;
 
-    my @files = <$dir/*.dat>;
+    $self->{Files} = [ map {File::Spec->rel2abs($_)} <$dir/*.dat> ];
 
     no warnings 'redefine';
     local *RT::Ticket::Load = sub {
@@ -253,9 +254,13 @@ sub Import {
         return $self->Id;
     };
 
-    for my $f (@files) {
-        open(my $fh, "<", $f) or die "Can't read $f: $!";
+    while (@{$self->{Files}}) {
+        $self->{Filename} = shift @{$self->{Files}};
+        open(my $fh, "<", $self->{Filename})
+            or die "Can't read $self->{Filename}: $!";
         while (not eof($fh)) {
+            $self->{Position} = tell($fh);
+
             my $loaded = Storable::fd_retrieve($fh);
             my ($class, $uid, $data) = @{$loaded};
 

commit a454d8bcba5b26bd7b45b1ca77806c6261c1f8f1
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 29 23:25:33 2011 -0500

    Serialize internal state when we hit an error

diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index cd456ca..67ee701 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -67,6 +67,7 @@ sub Init {
         PreserveTicketIds => 0,
         OriginalId        => undef,
         Progress          => undef,
+        Statefile         => undef,
         @_,
     );
 
@@ -94,7 +95,8 @@ sub Init {
         }
     }
 
-    $self->{Progress} = $args{Progress};
+    $self->{Progress}  = $args{Progress};
+    $self->{Statefile} = $args{Statefile};
 
     # Objects we've created
     $self->{UIDs} = {};
@@ -254,6 +256,8 @@ sub Import {
         return $self->Id;
     };
 
+    local $SIG{__DIE__} = sub { print STDERR "\n", @_; $self->SaveState; exit 1 };
+
     while (@{$self->{Files}}) {
         $self->{Filename} = shift @{$self->{Files}};
         open(my $fh, "<", $self->{Filename})
@@ -323,4 +327,29 @@ sub ObjectCount {
     return %{ $self->{ObjectCount} };
 }
 
+sub SaveState {
+    my $self = shift;
+
+    my %data;
+    unshift @{$self->{Files}}, $self->{Filename};
+    $self->{Seek} = $self->{Position};
+    $data{$_} = $self->{$_} for
+        qw/Filename Seek Position Files
+           ObjectCount
+           NewQueues NewCFs
+           SkipTransactions Pending
+           UIDs
+           OriginalId PreserveTicketIds
+          /;
+    Storable::nstore(\%data, $self->{Statefile});
+
+    print STDERR <<EOT;
+
+Importer state has been written to the file:
+    $self->{Statefile}
+
+It may be possible to resume the import by re-running rt-importer.
+EOT
+}
+
 1;
diff --git a/sbin/rt-importer b/sbin/rt-importer
index 9e96479..5a1765c 100755
--- a/sbin/rt-importer
+++ b/sbin/rt-importer
@@ -39,6 +39,7 @@ $import = RT::Migrate::Importer->new(
     PreserveTicketIds => $OPT{'preserve-tickets'},
     OriginalId        => $OPT{originalid},
     Progress          => $progress,
+    Statefile         => "$dir/partial-import",
 );
 
 my %counts = $import->Import( $dir );

commit 5d4514dbefc56585cc3fbb209e15ed2681780fad
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 30 00:01:07 2011 -0500

    Make ^C stop and write state when we hit an appropriate point in the loop

diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index 67ee701..d249a70 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -256,6 +256,7 @@ sub Import {
         return $self->Id;
     };
 
+    local $SIG{  INT  } = sub { $self->{INT} = 1 };
     local $SIG{__DIE__} = sub { print STDERR "\n", @_; $self->SaveState; exit 1 };
 
     while (@{$self->{Files}}) {
@@ -265,6 +266,9 @@ sub Import {
         while (not eof($fh)) {
             $self->{Position} = tell($fh);
 
+            # Stop when we're at a good stopping point
+            die "Caught interrupt, quitting.\n" if $self->{INT};
+
             my $loaded = Storable::fd_retrieve($fh);
             my ($class, $uid, $data) = @{$loaded};
 

commit 1b47d4ba1e70686afa064d99a973108ac0dfc8a2
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 29 23:26:01 2011 -0500

    When possible, restore state and resume where we left off

diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index d249a70..34ada91 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -248,6 +248,8 @@ sub Import {
 
     $self->{Files} = [ map {File::Spec->rel2abs($_)} <$dir/*.dat> ];
 
+    $self->RestoreState( $self->{Statefile} );
+
     no warnings 'redefine';
     local *RT::Ticket::Load = sub {
         my $self = shift;
@@ -263,6 +265,11 @@ sub Import {
         $self->{Filename} = shift @{$self->{Files}};
         open(my $fh, "<", $self->{Filename})
             or die "Can't read $self->{Filename}: $!";
+        if ($self->{Seek}) {
+            seek($fh, $self->{Seek}, 0)
+                or die "Can't seek to $self->{Seek} in $self->{Filename}";
+            $self->{Seek} = undef;
+        }
         while (not eof($fh)) {
             $self->{Position} = tell($fh);
 
@@ -331,6 +338,20 @@ sub ObjectCount {
     return %{ $self->{ObjectCount} };
 }
 
+sub RestoreState {
+    my $self = shift;
+    my ($statefile) = @_;
+    return unless $statefile and -f $statefile;
+
+    my $state = Storable::retrieve( $self->{Statefile} );
+    $self->{$_} = $state->{$_} for keys %{$state};
+    unlink $self->{Statefile};
+
+    print STDERR "Resuming partial import...\n";
+    sleep 2;
+    return 1;
+}
+
 sub SaveState {
     my $self = shift;
 

commit bbcf0da811cf14456b940f5589222ebc580158bb
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 29 23:28:10 2011 -0500

    Fix estimated completion time calculation, objects might be pre-imported
    
    Now that some data may already have been inserted into the system,
    adjust the rate calculations to count the starting number of objects
    inserted when calculating the average insertion rate.

diff --git a/lib/RT/Migrate.pm b/lib/RT/Migrate.pm
index 089e4f1..c90f24c 100644
--- a/lib/RT/Migrate.pm
+++ b/lib/RT/Migrate.pm
@@ -104,6 +104,7 @@ sub progress {
     my $last_time;
     my $start;
     my $left;
+    my $offset;
     return sub {
         my $obj = shift;
         my $now = Time::HiRes::time();
@@ -139,6 +140,7 @@ sub progress {
 
         my $total = 0;
         $total += $_ for values %counts;
+        $offset = $total unless defined $offset;
         print "\n", progress_bar(
             label => "Total",
             now   => $total,
@@ -148,7 +150,9 @@ sub progress {
         );
 
         # Time estimates
-        my $fraction = $max_objects ? $total/$max_objects : 0;
+        my $fraction = $max_objects
+            ? ($total - $offset)/($max_objects - $offset)
+            : 0;
         if ($fraction > 0.03) {
             if (defined $left) {
                 $left = 0.75 * $left
diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index 34ada91..e07b960 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -261,6 +261,7 @@ sub Import {
     local $SIG{  INT  } = sub { $self->{INT} = 1 };
     local $SIG{__DIE__} = sub { print STDERR "\n", @_; $self->SaveState; exit 1 };
 
+    $self->{Progress}->(undef) if $self->{Progress};
     while (@{$self->{Files}}) {
         $self->{Filename} = shift @{$self->{Files}};
         open(my $fh, "<", $self->{Filename})

commit fd8f12f17b0d7d1c87bfda244380543210cae45a
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Nov 29 23:59:09 2011 -0500

    Add percentage to progress bars

diff --git a/lib/RT/Migrate.pm b/lib/RT/Migrate.pm
index c90f24c..038b4c4 100644
--- a/lib/RT/Migrate.pm
+++ b/lib/RT/Migrate.pm
@@ -83,8 +83,8 @@ sub progress_bar {
     my $max_width = $args{cols} - 30;
     my $bar_width = int($max_width * $fraction);
 
-    return sprintf "%20s |%-" . $max_width . "s|\n",
-        $args{label}, $args{char} x $bar_width;
+    return sprintf "%20s |%-" . $max_width . "s| %3d%%\n",
+        $args{label}, $args{char} x $bar_width, $fraction*100;
 }
 
 sub progress {

commit 9895efb372676f17dffb9fa353d07660d7308830
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 30 15:17:45 2011 -0500

    Don't spew progressbars to non-terminals, and allow --quiet

diff --git a/sbin/rt-importer b/sbin/rt-importer
index 5a1765c..3246b25 100755
--- a/sbin/rt-importer
+++ b/sbin/rt-importer
@@ -16,6 +16,8 @@ use Time::HiRes qw//;
 my %OPT;
 GetOptions(
     \%OPT,
+    "quiet|q!",
+
     "preserve-tickets|t!",
     "originalid|i=s",
 );
@@ -27,7 +29,7 @@ $dir =~ s|/$||;
 my $import;
 
 my $progress;
-if (-f "$dir/rt-serialized") {
+if (-f "$dir/rt-serialized" and -t STDOUT and not $OPT{quiet}) {
     my $ref = Storable::retrieve("$dir/rt-serialized");
     $progress = RT::Migrate::progress(
         counts => sub { $import->ObjectCount },

commit a89cf9492c5823f30bae313b67bafbcadd4111bd
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 30 15:17:59 2011 -0500

    POD for rt-importer

diff --git a/sbin/rt-importer b/sbin/rt-importer
index 3246b25..830f127 100755
--- a/sbin/rt-importer
+++ b/sbin/rt-importer
@@ -49,3 +49,38 @@ print "Total object counts:\n";
 for (sort {$counts{$b} <=> $counts{$a}} keys %counts) {
     printf "%8d %s\n", $counts{$_}, $_;
 }
+
+=head1 NAME
+
+rt-importer - Import a serialized RT database on top of the current one
+
+=head1 SYNOPSIS
+
+    rt-importer path/to/export/directory
+
+This script is used to import the contents of a dump created by
+C<rt-serializer>.  It will create all of the objects in the dump in the
+current database; this may include users, queues, and tickets.
+
+It is possible to stop the import process with ^C; it can be later
+resumed by re-running the importer.
+
+=head2 OPTIONS
+
+=over
+
+=item B<--preserve-tickets>
+
+Attempt to preserve ticket numbers when inserting them into the
+database.  This will most probably fail if there are already tickets in
+the current database.
+
+=item B<--originalid> I<cfname>
+
+Places the original ticket organization and ID into a global custom
+field with the given name.  If no global ticket custom field with that
+name is found in the current database, it will create one.
+
+=back
+
+=cut

commit 63163059c174a7339c517387a87d537c35afb7d6
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 30 15:18:16 2011 -0500

    Use Pod::Usage for rt-importer

diff --git a/sbin/rt-importer b/sbin/rt-importer
index 830f127..409d0fc 100755
--- a/sbin/rt-importer
+++ b/sbin/rt-importer
@@ -16,13 +16,16 @@ use Time::HiRes qw//;
 my %OPT;
 GetOptions(
     \%OPT,
+    "help|?",
     "quiet|q!",
 
     "preserve-tickets|t!",
     "originalid|i=s",
-);
+) or Pod::Usage::pod2usage();
+
+Pod::Usage::pod2usage(-verbose => 1) if $OPT{help};
 
-die "Usage: rt-importer directory/\n" unless @ARGV == 1;
+Pod::Usage::pod2usage() unless @ARGV == 1;
 my ($dir) = @ARGV;
 $dir =~ s|/$||;
 

commit 5f32c76f42c6783d5c1547095d49f6cbd9a30b35
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Nov 30 15:18:25 2011 -0500

    Typo fix in rt-serializer POD

diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index cdff944..285c619 100755
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -255,7 +255,7 @@ thousand tickets, but optimal values will vary wildly depending on
 database configuration and size.  Values as low as 25 have provided
 speedups with smaller databases; if speed is a factor, experimenting
 with different C<--gc> values may be helpful.  Note that there are
-significant boundry condition changes in serialization rate, as the
+significant boundary condition changes in serialization rate, as the
 queue empties and fills, causing the time estimates to be rather
 imprecise near the start and end of the process.
 

commit d67a25760ec067a0fae015d3f27c7fce8439421e
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Dec 6 17:49:50 2011 -0500

    Queue could be 0 on templates and scrips

diff --git a/lib/RT/Scrip.pm b/lib/RT/Scrip.pm
index 0790c30..7fef46b 100644
--- a/lib/RT/Scrip.pm
+++ b/lib/RT/Scrip.pm
@@ -1002,7 +1002,7 @@ sub FindDependencies {
 
     $deps->Add( out => $self->ScripConditionObj );
     $deps->Add( out => $self->ScripActionObj );
-    $deps->Add( out => $self->QueueObj );
+    $deps->Add( out => $self->QueueObj ) if $self->QueueObj->Id;
     $deps->Add( out => $self->TemplateObj );
 }
 
diff --git a/lib/RT/Template.pm b/lib/RT/Template.pm
index 68fcc08..517c74f 100644
--- a/lib/RT/Template.pm
+++ b/lib/RT/Template.pm
@@ -949,7 +949,7 @@ sub FindDependencies {
 
     $self->SUPER::FindDependencies($walker, $deps);
 
-    $deps->Add( out => $self->QueueObj );
+    $deps->Add( out => $self->QueueObj ) if $self->QueueObj->Id;
 }
 
 sub PreInflate {

commit 230317a1612b31092e42268028aec05e49125ba1
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Dec 6 18:05:00 2011 -0500

    OCFs may have an id of 0 if they are applied globally

diff --git a/lib/RT/ObjectCustomField.pm b/lib/RT/ObjectCustomField.pm
index 444c7d3..7c24d45 100644
--- a/lib/RT/ObjectCustomField.pm
+++ b/lib/RT/ObjectCustomField.pm
@@ -415,10 +415,12 @@ sub FindDependencies {
 
     $deps->Add( out => $self->CustomFieldObj );
 
-    my $class = $self->CustomFieldObj->RecordClassFromLookupType;
-    my $obj = $class->new( $self->CurrentUser );
-    $obj->Load( $self->ObjectId );
-    $deps->Add( out => $obj );
+    if ($self->ObjectId) {
+        my $class = $self->CustomFieldObj->RecordClassFromLookupType;
+        my $obj = $class->new( $self->CurrentUser );
+        $obj->Load( $self->ObjectId );
+        $deps->Add( out => $obj );
+    }
 }
 
 sub Serialize {

commit 2da552fc87678c5f179b0e632e82c4064f0a1138
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Dec 7 15:30:40 2011 -0500

    Make sure to include deleted rows in estimates

diff --git a/sbin/rt-serializer b/sbin/rt-serializer
index 285c619..7757be4 100755
--- a/sbin/rt-serializer
+++ b/sbin/rt-serializer
@@ -104,6 +104,7 @@ sub estimate {
         print "Estimating $class count...";
         my $collection = $class . "s";
         my $objs = $collection->new( RT->SystemUser );
+        $objs->FindAllRows;
         $objs->UnLimit;
         $objs->{allow_deleted_search} = 1 if $class eq "RT::Ticket";
         $e{$class} = $objs->Count;

commit 5999a10e8c207bde3e94f88a5655ef3c5280f887
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Thu Dec 22 19:32:56 2011 -0500

    Store the ticket id, not the effective ticket's id

diff --git a/lib/RT/Ticket.pm b/lib/RT/Ticket.pm
index 4849ea6..3143884 100644
--- a/lib/RT/Ticket.pm
+++ b/lib/RT/Ticket.pm
@@ -4219,7 +4219,7 @@ sub Serialize {
     $store{EffectiveId} = \($obj->UID);
 
     # Shove the ID back in, in case we want to preserve it during import
-    $store{id} = $obj->Id;
+    $store{id} = $self->id;
 
     return %store;
 }

commit 0f555d02e14b8f1d2a6d241e27d1920d51bd4c53
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue Jan 17 16:58:28 2012 -0500

    Allow links to come before the tickets they reference
    
    While the serialization process can _nearly_ guarantee that no object is
    serialized before its dependencies, there are some times when it cannot
    uphold this -- specifically, involving links to deleted tickets, for
    instance.  In these cases, use the Postpone machinery (extended to allow
    for ->URI) to update the link object later.

diff --git a/lib/RT/Link.pm b/lib/RT/Link.pm
index e88892d..541995f 100644
--- a/lib/RT/Link.pm
+++ b/lib/RT/Link.pm
@@ -473,10 +473,21 @@ sub PreInflate {
         my $uid_ref = delete $data->{$dir};
         next unless $uid_ref and ref $uid_ref;
 
-        my $uid = ${ $uid_ref };
-        my $obj = $importer->LookupObj( $uid );
-        $data->{$dir} = $obj->URI;
-        $data->{"Local$dir"} = $obj->Id if $obj->isa("RT::Ticket");
+        my $to_uid = ${ $uid_ref };
+        my $obj = $importer->LookupObj( $to_uid );
+        if ($obj) {
+            $data->{$dir} = $obj->URI;
+            $data->{"Local$dir"} = $obj->Id if $obj->isa("RT::Ticket");
+        } else {
+            $data->{$dir} = "";
+            $importer->Postpone(
+                for => $to_uid,
+                uid => $uid,
+                uri => $dir,
+                column => ($to_uid =~ /RT::Ticket/ ? "Local$dir" : undef),
+            );
+        }
+
     }
 
     return $class->SUPER::PreInflate( $importer, $uid, $data );
diff --git a/lib/RT/Migrate/Importer.pm b/lib/RT/Migrate/Importer.pm
index e07b960..d690911 100644
--- a/lib/RT/Migrate/Importer.pm
+++ b/lib/RT/Migrate/Importer.pm
@@ -129,11 +129,15 @@ sub Resolve {
         $obj->__Set(
             Field => $ref->{column},
             Value => $id,
-        );
+        ) if defined $ref->{column};
         $obj->__Set(
             Field => $ref->{classcolumn},
             Value => $class,
-        ) if $ref->{classcolumn};
+        ) if defined $ref->{classcolumn};
+        $obj->__Set(
+            Field => $ref->{uri},
+            Value => $self->LookupObj($uid)->URI,
+        ) if defined $ref->{uri};
     }
     delete $self->{Pending}{$uid};
 }
@@ -163,6 +167,7 @@ sub Postpone {
         uid         => undef,
         column      => undef,
         classcolumn => undef,
+        uri         => undef,
         @_,
     );
     my $uid = delete $args{for};

-----------------------------------------------------------------------


More information about the Rt-commit mailing list