[Rt-commit] rt branch, 4.4/merged-fulltext, updated. rt-4.4.1-326-g9ac16bf

Aaron Kondziela aaron at bestpractical.com
Thu May 4 20:06:19 EDT 2017


The branch, 4.4/merged-fulltext has been updated
       via  9ac16bfaa266c9d462067e959ff36619da4c550b (commit)
      from  e6d618436fcb25dbf7b4b5c53df640ca2e9e2da3 (commit)

Summary of changes:
 .gitignore                              |   1 +
 configure.ac                            |   1 +
 docs/UPGRADING-4.4                      |  22 +++-
 etc/upgrade/4.4.2/schema.mysql          |   2 -
 etc/upgrade/4.4.2/schema.mysql.template |   7 +
 etc/upgrade/upgrade-fulltext.in         | 219 ++++++++++++++++++++++++++++++++
 sbin/rt-setup-database.in               |  13 ++
 7 files changed, 258 insertions(+), 7 deletions(-)
 delete mode 100644 etc/upgrade/4.4.2/schema.mysql
 create mode 100644 etc/upgrade/4.4.2/schema.mysql.template
 create mode 100644 etc/upgrade/upgrade-fulltext.in

- Log -----------------------------------------------------------------
commit 9ac16bfaa266c9d462067e959ff36619da4c550b
Author: Aaron Kondziela <aaron at bestpractical.com>
Date:   Thu May 4 20:06:57 2017 -0400

    Add schema upgrade templates and upgrade-fulltext tool

diff --git a/.gitignore b/.gitignore
index 35850b0..9a9d1a3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,7 @@
 /etc/upgrade/upgrade-sla
 /etc/upgrade/upgrade-assets
 /etc/upgrade/vulnerable-passwords
+/etc/upgrade/upgrade-fulltext
 /lib/RT/Generated.pm
 /Makefile
 /t/data/gnupg/keyrings/random_seed
diff --git a/configure.ac b/configure.ac
index 6b9e88c..4825693 100755
--- a/configure.ac
+++ b/configure.ac
@@ -463,6 +463,7 @@ AC_CONFIG_FILES([
                  etc/upgrade/upgrade-assets
                  etc/upgrade/vulnerable-passwords
                  etc/upgrade/upgrade-sla
+                 etc/upgrade/upgrade-fulltext
                  sbin/rt-ldapimport
                  sbin/rt-attributes-viewer
                  sbin/rt-preferences-viewer
diff --git a/docs/UPGRADING-4.4 b/docs/UPGRADING-4.4
index e725572..f4cb654 100644
--- a/docs/UPGRADING-4.4
+++ b/docs/UPGRADING-4.4
@@ -525,9 +525,21 @@ AfterUpdateCustomFieldValue. It will be removed in RT 4.6.
 
 A fix for using the MySQL full text index (F<docs/full_text_indexing.pod>) with
 merged tickets requires changes to the database schema and processing to update
-the data. After you upgrade the RT code, you must drop and re-create the index.
-To do this, run the setup command as you would during initial configuration of
-the fulltext index feature:
+the data. After you upgrade the RT code, you can drop and re-create the index,
+or you can run a tool to process the existing index.
+
+B<If you have a large number of tickets, you should run the F<upgrade-database>>
+B<tool, as re-creating the index can take a very long time.>
+
+To process an existing index after running C<make upgrade-database>, run this
+tool from your RT base directory:
+
+    ./etc/upgrade/upgrade-fulltext --verbose
+
+For additional options you can use the C<--help> flag.
+
+To drop and re-create the index, run the setup command as you would during
+initial configuration of the fulltext index feature:
 
     ./sbin/rt-setup-fulltext-index
 
@@ -535,8 +547,8 @@ Please be aware that the F<sbin/rt-fulltext-indexer> tool which keeps the index
 updated requires the new schema. If you have that tool set to run automatically
 via cron or another means, it will fail after the RT upgrade until you have the
 database updated as instructed above. We recommend that you temporarily disable
-automatic running of F<rt-fulltext-indexer> until after you complete this
-upgrade.
+automatic running of F<rt-fulltext-indexer> until after you complete these
+upgrade steps.
 
 =back
 
diff --git a/etc/upgrade/4.4.2/schema.mysql b/etc/upgrade/4.4.2/schema.mysql
deleted file mode 100644
index e359ded..0000000
--- a/etc/upgrade/4.4.2/schema.mysql
+++ /dev/null
@@ -1,2 +0,0 @@
-ALTER TABLE CustomFields ADD COLUMN UniqueValues int2 NOT NULL DEFAULT 0;
-ALTER TABLE CustomFields ADD COLUMN CanonicalizeClass varchar(64) CHARACTER SET ascii NULL;
diff --git a/etc/upgrade/4.4.2/schema.mysql.template b/etc/upgrade/4.4.2/schema.mysql.template
new file mode 100644
index 0000000..4d63039
--- /dev/null
+++ b/etc/upgrade/4.4.2/schema.mysql.template
@@ -0,0 +1,7 @@
+ALTER TABLE CustomFields ADD COLUMN UniqueValues int2 NOT NULL DEFAULT 0;
+ALTER TABLE CustomFields ADD COLUMN CanonicalizeClass varchar(64) CHARACTER SET ascii NULL;
+
+ALTER TABLE {RT->Config->Get('FullTextSearch')->{'Table'}} ADD COLUMN TicketId INT UNSIGNED;
+ALTER TABLE {RT->Config->Get('FullTextSearch')->{'Table'}} ADD COLUMN EffectiveId INT UNSIGNED;
+ALTER TABLE {RT->Config->Get('FullTextSearch')->{'Table'}} ADD INDEX TicketIdKey (TicketId);
+ALTER TABLE {RT->Config->Get('FullTextSearch')->{'Table'}} ADD INDEX EffectiveIdKey (EffectiveId);
diff --git a/etc/upgrade/upgrade-fulltext.in b/etc/upgrade/upgrade-fulltext.in
new file mode 100644
index 0000000..5bafde7
--- /dev/null
+++ b/etc/upgrade/upgrade-fulltext.in
@@ -0,0 +1,219 @@
+#!@PERL@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2017 Best Practical Solutions, LLC
+#                                          <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions,  LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+use 5.10.1;
+use strict;
+use warnings;
+
+use lib "@LOCAL_LIB_PATH@";
+use lib "@RT_LIB_PATH@";
+
+use RT -init;
+use RT::Interface::CLI ();
+use Getopt::Long qw(GetOptions);
+
+my %OPT = ();
+GetOptions( \%OPT,
+    "help|h!",
+    "verbose|v!",
+    "chunksize=i",
+);
+
+$OPT{'chunksize'} ||= 10000;
+
+RT::Interface::CLI->ShowHelp if $OPT{help};
+
+=pod
+
+=head1 NAME
+
+upgrade-fulltext - Upgrade full text index schema as required by RT 4.4.2
+
+=head1 USAGE
+
+    # Run upgrade after upgrading RT to 4.4.2
+    # Read docs/UPGRADING-4.4 first
+    # From rt base dir:
+
+    ./etc/upgrade/upgrade-fulltext [-v|--verbose] [--chunksize <number>]
+    ./etc/upgrade/upgrade-fulltext [-h|--help] - print help text
+
+        --verbose   : print additional detailed output
+
+        --chunksize : Select <number> of rows at a time from index.
+                      Default is 10000
+
+        --help      : Print help text
+
+=head1 DESCRIPTION
+
+When upgrading to RT 4.4.2 you must run this command to upgrade the full text
+database schema. You only need to do this if fulltext indexing is active in
+your config, and you are using MySQL. See F<docs/UPGRADING-4.4> for more
+details.
+
+=cut
+
+sub verbose {
+    print STDERR (shift . (shift ? "" : "\n")) if $OPT{verbose};
+}
+
+verbose("Checking locks");
+use Fcntl ':flock';
+open (my $indexer_cmd_fh, '<', $RT::BasePath . '/sbin/rt-fulltext-indexer')
+    or die "Error: $!";
+if ( !flock $indexer_cmd_fh, LOCK_EX | LOCK_NB ) {
+    print STDERR "rt-fulltext-indexer is running.\nYou should disable it before running this tool.\n";
+    exit 1;
+}
+if ( !flock main::DATA, LOCK_EX | LOCK_NB ) {
+    print STDERR "Another instance of this command is running; aborting.\n";
+    exit 2;
+}
+
+verbose("Checking database type and fulltext configuration");
+my $db_type = RT->Config->Get('DatabaseType');
+unless ($db_type eq 'mysql') {
+    print STDERR "This command is only for mysql databases; aborting\n";
+    exit 3;
+}
+
+my $fts_conf = RT->Config->Get('FullTextSearch') || {};
+unless ($fts_conf->{'Enable'} and $fts_conf->{'Indexed'} and not $fts_conf->{'Sphinx'}) {
+    print STDERR "Your FullTextSearch configuration doesn't need this command; aborting\n";
+    exit 4;
+}
+
+my $index_table = $fts_conf->{'Table'};
+my $dbh = $RT::Handle->dbh;
+
+# Check if upgrade-database has been run first
+{
+    my $db_is_updated;
+    my $sth = $dbh->prepare("SHOW COLUMNS FROM $index_table");
+    $sth->execute();
+
+    while (my $row = $sth->fetchrow_hashref() ) {
+        $db_is_updated = 1 if $row->{field} eq 'EffectiveId';
+    }
+
+    die("Error: You must update your MySQL schema before running this. See docs/UPGRADING-4.4 for details.")
+        unless $db_is_updated;
+}
+
+my $chunksize = $OPT{'chunksize'};  # rows to select and process at once
+my $reporting_cadence = $chunksize; # how often to spit out a numeric progres indicator
+
+my $count_st = "SELECT COUNT(id) FROM $index_table WHERE TicketId IS NULL OR EffectiveId IS NULL";
+
+my $empty_st = "SELECT id AS attachmentid FROM $index_table "
+    . "WHERE (TicketId IS NULL OR EffectiveId IS NULL) "
+    . "AND id > ? "
+    . "ORDER BY id ASC "
+    . "LIMIT $chunksize"
+    ;
+
+my $ticket_st = "SELECT Attachments.id AS attachmentid, Tickets.id AS ticketid, Tickets.EffectiveId AS effectiveid "
+    . "FROM Attachments "
+    . "LEFT JOIN Transactions ON Attachments.TransactionId = Transactions.id "
+    . "LEFT JOIN Tickets ON Transactions.ObjectId = Tickets.id "
+    . "WHERE Attachments.id = ?"
+    ;
+
+my $update_st = "UPDATE $index_table SET TicketId=?, EffectiveId=? WHERE id=?";
+
+verbose("Preparing SQL");
+my $count_sth = $dbh->prepare($count_st);
+my $empty_sth = $dbh->prepare($empty_st);
+my $ticket_sth = $dbh->prepare($ticket_st);
+my $update_sth = $dbh->prepare($update_st);
+
+$count_sth->execute() or die("Could not execute count_st: " . $dbh->errstr);
+my $countrow = $count_sth->fetchrow_arrayref;
+my $todo = $$countrow[0]; # total count of rows that need processing
+
+my $last = 0;
+my $empty_rows_found;
+my $chunks_processed = 0;
+
+print "Processing $todo rows in $index_table. This may take some time on large databases.\n";
+verbose("Processing null attachment entries in chunks of $chunksize (one dot per chunk)");
+
+{
+    $empty_sth->execute($last) or die("Could not execute empty_st: " . $dbh->errstr);
+    $empty_rows_found = 0;
+
+    while (my $row = $empty_sth->fetchrow_hashref) {
+        $empty_rows_found++;
+
+        $ticket_sth->execute( $row->{'attachmentid'} )
+            or die ("Could not execute ticket_sth: " . $dbh->errstr);
+        my $tik = $ticket_sth->fetchrow_hashref;
+
+        $update_sth->execute( $tik->{'ticketid'}, $tik->{'effectiveid'}, $row->{'attachmentid'} )
+            or die ("Could not execute update_sth: " . $dbh->errstr);
+
+        $last = $row->{'attachmentid'};
+
+        if (($chunksize * $chunks_processed + $empty_rows_found) % $reporting_cadence == 0) {
+            verbose("");
+            print "Processed " . ($chunksize * $chunks_processed + $empty_rows_found) . " of $todo\n";
+        }
+    }
+
+    if ($empty_rows_found == $chunksize) {
+        verbose(".", 1);
+        $chunks_processed++;
+        redo;
+    }
+}
+verbose("\nDone");
+
+print "OK: Processed " . ($chunksize * $chunks_processed + $empty_rows_found) . " of $todo entries in $index_table\n\n";
+
+__DATA__
diff --git a/sbin/rt-setup-database.in b/sbin/rt-setup-database.in
index 6cd7f35..bf86658 100644
--- a/sbin/rt-setup-database.in
+++ b/sbin/rt-setup-database.in
@@ -72,6 +72,7 @@ BEGIN { # BEGIN RT CMD BOILERPLATE
 use Term::ReadKey;
 use Getopt::Long;
 use Data::GUID;
+use Text::Template;
 
 $| = 1; # unbuffer all output.
 
@@ -553,6 +554,18 @@ sub action_upgrade {
 
         my %tmp = (%args, datadir => "$base_dir/$v", datafile => undef, backcompat => \@back);
 
+        if ( -e "$base_dir/$v/schema.$db_type.template" ) {
+            my $template = Text::Template->new(
+                TYPE => 'file',
+                SOURCE => "$base_dir/$v/schema.$db_type.template",
+            );
+            open(my $fh, '>', "$base_dir/$v/schema.$db_type")
+                or die "Can't open $base_dir/$v/schema.$db_type for writing template output: $!";
+            my $template_out = $template->fill_in(OUTPUT => $fh)
+                or die "Can't fill in template: $Text::Template::ERROR";
+            close $fh;
+        }
+
         if ( -e "$base_dir/$v/schema.$db_type" ) {
             ( $ret, $msg ) = action_schema( %tmp );
             return ( $ret, $msg ) unless $ret;

-----------------------------------------------------------------------


More information about the rt-commit mailing list