[Rt-commit] rt branch, 4.4/serialize-json-initialdata, updated. rt-4.4.1-429-g0b46105
Shawn Moore
shawn at bestpractical.com
Thu Mar 23 15:25:17 EDT 2017
The branch, 4.4/serialize-json-initialdata has been updated
via 0b46105d9335a306309fc3f051d1eff94b083ebe (commit)
via aff375312ed8041c678f6bb94b0c2fe37c58f59f (commit)
via c8bb7354ce15fab7eac9a7799c474c1aef8a1d9a (commit)
via a97199da5eceea4ed9177270fc5f0ce8a1e073e9 (commit)
via 53a375c2a64e636f6c769ce9970b06a7d3c8587d (commit)
from b28da74ced454cc59311a500bb684b6d7a09fb15 (commit)
Summary of changes:
.gitignore | 1 +
Makefile.in | 1 +
configure.ac | 1 +
lib/RT/Migrate/Serializer.pm | 5 +-
lib/RT/Migrate/Serializer/JSON.pm | 9 ++
sbin/rt-merge-initialdata.in | 190 ++++++++++++++++++++++++++++++++++++++
sbin/rt-serializer.in | 3 +-
7 files changed, 206 insertions(+), 4 deletions(-)
create mode 100644 sbin/rt-merge-initialdata.in
- Log -----------------------------------------------------------------
commit 53a375c2a64e636f6c769ce9970b06a7d3c8587d
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Thu Mar 23 18:05:11 2017 +0000
Delete empty Values => [] from custom fields
Especially important for CF types that don't use values
diff --git a/lib/RT/Migrate/Serializer/JSON.pm b/lib/RT/Migrate/Serializer/JSON.pm
index 5eb0a30..8c9d16d 100644
--- a/lib/RT/Migrate/Serializer/JSON.pm
+++ b/lib/RT/Migrate/Serializer/JSON.pm
@@ -237,6 +237,7 @@ sub _CanonicalizeManyToMany {
primary_key => 'ApplyTo',
add_to_primary => undef,
sort_uniq => 0,
+ delete_empty => 0,
finalize => undef,
canonicalize_object => sub { $_->{ObjectId} },
@_,
@@ -249,6 +250,7 @@ sub _CanonicalizeManyToMany {
my $primary_key = $args{primary_key};
my $add_to_primary = $args{add_to_primary};
my $sort_uniq = $args{sort_uniq};
+ my $delete_empty = $args{delete_empty};
my $finalize = $args{finalize};
my $canonicalize_object = $args{canonicalize_object};
@@ -273,6 +275,10 @@ sub _CanonicalizeManyToMany {
@{ $primary->{$primary_key} };
}
+ if ($delete_empty) {
+ delete $primary->{$primary_key} if !@{ $primary->{$primary_key} };
+ }
+
if ($finalize) {
$finalize->($primary);
}
@@ -455,6 +461,7 @@ sub CanonicalizeObjects {
object_sorter => 'Name',
primary_class => 'RT::CustomField',
primary_key => 'Values',
+ delete_empty => 1,
canonicalize_object => sub {
my %object = %$_;
return if $object{Disabled} && !$self->{FollowDisabled};
commit a97199da5eceea4ed9177270fc5f0ce8a1e073e9
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Thu Mar 23 18:17:08 2017 +0000
No need to assign Verbose separately
diff --git a/lib/RT/Migrate/Serializer.pm b/lib/RT/Migrate/Serializer.pm
index 4baeda0..d402767 100644
--- a/lib/RT/Migrate/Serializer.pm
+++ b/lib/RT/Migrate/Serializer.pm
@@ -81,8 +81,6 @@ sub Init {
@_,
);
- $self->{Verbose} = delete $args{Verbose};
-
$self->{$_} = delete $args{$_}
for qw/
AllUsers
@@ -97,6 +95,7 @@ sub Init {
Clone
Incremental
Sync
+ Verbose
/;
$self->{Clone} = 1 if $self->{Incremental};
commit c8bb7354ce15fab7eac9a7799c474c1aef8a1d9a
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Thu Mar 23 18:22:23 2017 +0000
Include export options under --sync
This way we can easily regenerate a fresh initialdata if needed
diff --git a/lib/RT/Migrate/Serializer.pm b/lib/RT/Migrate/Serializer.pm
index d402767..f513a12 100644
--- a/lib/RT/Migrate/Serializer.pm
+++ b/lib/RT/Migrate/Serializer.pm
@@ -81,7 +81,7 @@ sub Init {
@_,
);
- $self->{$_} = delete $args{$_}
+ $self->{$_} = $self->{ExportOptions}{$_} = delete $args{$_}
for qw/
AllUsers
AllGroups
diff --git a/lib/RT/Migrate/Serializer/JSON.pm b/lib/RT/Migrate/Serializer/JSON.pm
index 8c9d16d..1d7e524 100644
--- a/lib/RT/Migrate/Serializer/JSON.pm
+++ b/lib/RT/Migrate/Serializer/JSON.pm
@@ -611,6 +611,8 @@ sub WriteFile {
}
}
+ $output{ExportOptions} = $self->{ExportOptions} if $self->{Sync};
+
print { $self->{Filehandle} } $self->JSON->encode(\%output);
}
diff --git a/sbin/rt-serializer.in b/sbin/rt-serializer.in
index 5fee4d0..8455dd3 100644
--- a/sbin/rt-serializer.in
+++ b/sbin/rt-serializer.in
@@ -378,7 +378,8 @@ C<docs/incremental-export/>.
When exporting an initialdata, record ids are ordinarily excluded. Pass
C<--sync> to include record ids if you intend to use this for sync
-rather than creating a generic initialdata.
+rather than creating a generic initialdata. This also includes the export
+options in the initialdata for later reuse.
=item B<--gc> I<n>
commit aff375312ed8041c678f6bb94b0c2fe37c58f59f
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Thu Mar 23 19:17:29 2017 +0000
WIP for a new rt-merge-initialdata
diff --git a/.gitignore b/.gitignore
index 35850b0..eef91d7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -42,6 +42,7 @@
/sbin/rt-validator
/sbin/rt-validate-aliases
/sbin/rt-serializer
+/sbin/rt-merge-initialdata
/sbin/rt-importer
/sbin/rt-ldapimport
/sbin/standalone_httpd
diff --git a/Makefile.in b/Makefile.in
index 464bda0..a8944fa 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -148,6 +148,7 @@ SYSTEM_BINARIES = rt-attributes-viewer \
rt-fulltext-indexer \
rt-importer \
rt-ldapimport \
+ rt-merge-initialdata \
rt-preferences-viewer \
rt-serializer \
rt-server \
diff --git a/configure.ac b/configure.ac
index 6b9e88c..2194b6f 100755
--- a/configure.ac
+++ b/configure.ac
@@ -484,6 +484,7 @@ AC_CONFIG_FILES([
sbin/rt-setup-fulltext-index
sbin/rt-fulltext-indexer
sbin/rt-serializer
+ sbin/rt-merge-initialdata
sbin/rt-importer
bin/rt-crontool
bin/rt-mailgate
diff --git a/sbin/rt-merge-initialdata.in b/sbin/rt-merge-initialdata.in
new file mode 100644
index 0000000..cca4dd5
--- /dev/null
+++ b/sbin/rt-merge-initialdata.in
@@ -0,0 +1,171 @@
+#!@PERL@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2017 Best Practical Solutions, LLC
+# <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+use strict;
+use warnings;
+
+# fix lib paths, some may be relative
+BEGIN {
+ require File::Spec;
+ my @libs = ("@RT_LIB_PATH@", "@LOCAL_LIB_PATH@");
+ my $bin_path;
+
+ for my $lib (@libs) {
+ unless ( File::Spec->file_name_is_absolute($lib) ) {
+ unless ($bin_path) {
+ if ( File::Spec->file_name_is_absolute(__FILE__) ) {
+ $bin_path = ( File::Spec->splitpath(__FILE__) )[1];
+ }
+ else {
+ require FindBin;
+ no warnings "once";
+ $bin_path = $FindBin::Bin;
+ }
+ }
+ $lib = File::Spec->catfile( $bin_path, File::Spec->updir, $lib );
+ }
+ unshift @INC, $lib;
+ }
+
+}
+
+use RT;
+RT::LoadConfig();
+RT::Init();
+
+use JSON ();
+my $JSON = JSON->new->canonical;
+
+die "usage: $0 base edited\n" unless @ARGV == 2;
+my ($base_file, $edited_file) = @ARGV;
+
+my $base_records = slurp_json($base_file);
+my $edited_records = slurp_json($edited_file);
+
+my $export_options = delete $edited_records->{ExportOptions}
+ or die "Required metadata ExportOptions not present in $edited_file. Did you pass --sync to rt-serializer?";
+delete $base_records->{ExportOptions};
+
+my @record_types = qw/Groups Users Members ACL Queues Classes
+ ScripActions ScripConditions Templates
+ CustomFields CustomRoles Scrips
+ Catalogs Assets Articles/;
+
+for my $type (@record_types) {
+ my ($new_records, $updated_records, $deleted_records) = find_differences(
+ $base_records->{$type},
+ $edited_records->{$type},
+ $type,
+ );
+
+ my $collection_class = "RT::$type";
+ my $record_class = $collection_class->RecordClass;
+
+ for my $new (@$new_records) {
+ my $record = $record_class->new(RT->SystemUser);
+ }
+
+ for my $deleted (@$deleted_records) {
+ my $record = $record_class->new(RT->SystemUser);
+ $record->Load($deleted->{id});
+ }
+
+ for (@$updated_records) {
+ my ($base, $edited) = @_;
+ my $record = $record_class->new(RT->SystemUser);
+ $record->Load($base->{id});
+ }
+}
+
+sub find_differences {
+ my $base_records = shift;
+ my $edited_records = shift;
+ my $type = shift;
+
+ my (@new, @deleted, @updated);
+ my (%base_by_id, %edited_by_id);
+
+ for my $base_record (@$base_records) {
+ my $id = $base_record->{id};
+
+ if (!$id) {
+ die "Missing id for this $type record in $base_file: " . encode_json($base_record);
+ }
+ $base_by_id{$id} = $base_record;
+ }
+
+ for my $edited_record (@$edited_records) {
+ my $id = $edited_record->{id};
+
+ if (!$id) {
+ push @new, $edited_record;
+ }
+ elsif (!$base_by_id{$id}) {
+ die "$type record in $edited_file has id ($id) that doesn't correspond with a record in $base_file: " . $JSON->encode($edited_record);
+ }
+ else {
+ my $base_record = delete $base_by_id{$id};
+ next if $JSON->encode($base_record) eq $JSON->encode($edited_record);
+ push @updated, [ $base_record => $edited_record ];
+ }
+ }
+
+ for my $base_record (values %base_by_id) {
+ push @deleted, $base_record;
+ }
+
+ return (\@new, \@updated, \@deleted);
+}
+
+sub slurp_json {
+ my $file = shift;
+ local $/;
+ open (my $f, '<', $file)
+ or die "Cannot open initialdata file '$file' for read: $@";
+ return $JSON->decode(scalar <$f>);
+}
commit 0b46105d9335a306309fc3f051d1eff94b083ebe
Author: Shawn M Moore <shawn at bestpractical.com>
Date: Thu Mar 23 19:21:27 2017 +0000
Implement record disabling/deleting
diff --git a/sbin/rt-merge-initialdata.in b/sbin/rt-merge-initialdata.in
index cca4dd5..b569b30 100644
--- a/sbin/rt-merge-initialdata.in
+++ b/sbin/rt-merge-initialdata.in
@@ -111,8 +111,27 @@ for my $type (@record_types) {
}
for my $deleted (@$deleted_records) {
+ my $id = $deleted->{id};
my $record = $record_class->new(RT->SystemUser);
- $record->Load($deleted->{id});
+ $record->Load($id);
+
+ my ($ok, $msg);
+ if ($record->can('SetDisabled') || $record->_Accessible('Disabled', 'write')) {
+ ($ok, $msg) = $record->SetDisabled(1);
+ }
+ elsif ($record->can('Delete')) {
+ ($ok, $msg) = $record->Delete;
+ }
+ else {
+ die "No method to delete $record_class #$id";
+ }
+
+ if ($ok) {
+ RT->Logger->debug("Deleted $record_class $id: $msg");
+ }
+ else {
+ RT->Logger->error("Unable to delete $record_class $id: $msg");
+ }
}
for (@$updated_records) {
-----------------------------------------------------------------------
More information about the rt-commit
mailing list