[Rt-commit] rt branch, 4.2/fts-indexer-errors, created. rt-4.2.10-194-ga3e7081

Alex Vandiver alexmv at bestpractical.com
Mon Mar 16 19:45:16 EDT 2015


The branch, 4.2/fts-indexer-errors has been created
        at  a3e7081db3be9716563b00c139c658f9bc9d4007 (commit)

- Log -----------------------------------------------------------------
commit 5eb8b53936c7e0b5b4f1ed68d94817a61b5379ce
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Mar 13 14:32:29 2015 -0400

    Refactor shared error handler for postgres error conditions

diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index bd55adb..72e1faf 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -259,14 +259,29 @@ sub process_mysql {
 
 
 sub process_pg {
+    my $error_handling = sub {
+        my ($id) = @_;
+        my $dbh = $RT::Handle->dbh;
+        if ( $dbh->err == 7 && $dbh->state eq '54000' ) {
+            warn "Attachment $id cannot be indexed. Most probably it contains too many unique words. ".
+                "Error: ". $dbh->errstr;
+        } elsif ( $dbh->err == 7 && $dbh->state eq '22021' ) {
+            warn "Attachment $id cannot be indexed. Most probably it contains invalid UTF8 bytes. ".
+                "Error: ". $dbh->errstr;
+        } else {
+            die "Attachment $id cannot be indexed: " . $dbh->errstr;
+        }
+    };
+
     if ( $fts_config->{'Table'} ne 'Attachments' ) {
-        process_pg_insert();
+        process_pg_insert( $error_handling );
     } else {
-        process_pg_update();
+        process_pg_update( $error_handling );
     }
 }
 
 sub process_pg_insert {
+    my ($error_handling) = @_;
     my $dbh = $RT::Handle->dbh;
     my $table = $fts_config->{'Table'};
     my $column = $fts_config->{'Column'};
@@ -278,22 +293,12 @@ sub process_pg_insert {
             return "INSERT INTO $table($column, id) VALUES "
                 . join(", ", ("(TO_TSVECTOR(?),?)") x $n);
         },
-        sub {
-            my ($id) = @_;
-            if ( $dbh->err == 7 && $dbh->state eq '54000' ) {
-                warn "Attachment $id cannot be indexed. Most probably it contains too many unique words. ".
-                  "Error: ". $dbh->errstr;
-            } elsif ( $dbh->err == 7 && $dbh->state eq '22021' ) {
-                warn "Attachment $id cannot be indexed. Most probably it contains invalid UTF8 bytes. ".
-                  "Error: ". $dbh->errstr;
-            } else {
-                die "Attachment $id cannot be indexed: " . $dbh->errstr;
-            }
-        }
+        $error_handling,
     );
 }
 
 sub process_pg_update {
+    my ($error_handling) = @_;
     my $dbh = $RT::Handle->dbh;
     my $column = $fts_config->{'Column'};
 
@@ -332,15 +337,8 @@ sub process_pg_update {
         for (@insert) {
             my ($content, $id) = ($_->[0], $_->[1]);
             next if eval { $sth->execute( $content, $id ) };
-            if ( $dbh->err == 7  && $dbh->state eq '54000' ) {
-                warn "Attachment $id cannot be indexed. Most probably it contains too many unique words. ".
-                  "Error: ". $dbh->errstr;
-            } elsif ( $dbh->err == 7 && $dbh->state eq '22021' ) {
-                warn "Attachment $id cannot be indexed. Most probably it contains invalid UTF8 bytes. ".
-                  "Error: ". $dbh->errstr;
-            } else {
-                die "Attachment $id cannot be indexed: " . $dbh->errstr;
-            }
+
+            $error_handling->($id);
 
             # If this was a semi-expected error, insert an empty
             # tsvector, so we count this row as "indexed" for

commit 859d5c4987018209eb036bda0a1f4e09a17ed248
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Mar 13 18:19:47 2015 -0400

    Catch "invalid memory alloc request size ..." errors

diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index 72e1faf..b4dbf20 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -268,6 +268,9 @@ sub process_pg {
         } elsif ( $dbh->err == 7 && $dbh->state eq '22021' ) {
             warn "Attachment $id cannot be indexed. Most probably it contains invalid UTF8 bytes. ".
                 "Error: ". $dbh->errstr;
+        } elsif ( $dbh->err == 7 && $dbh->state eq 'XX000' ) {
+            warn "Attachment $id cannot be indexed; it is too large.  ".
+                "Error: ". $dbh->errstr;
         } else {
             die "Attachment $id cannot be indexed: " . $dbh->errstr;
         }

commit ecb344294941fc0ba5ecf38d3b776083daa33468
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Mar 16 13:14:02 2015 -0400

    Hide errors from the DBH that will get their own via the indexer

diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index b4dbf20..a326de7 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -275,6 +275,11 @@ sub process_pg {
             die "Attachment $id cannot be indexed: " . $dbh->errstr;
         }
     };
+    my $dbh = $RT::Handle->dbh; Scalar::Util::weaken($dbh);
+    $dbh->{HandleError} = sub {
+        return 1 if $dbh->err == 7 and $dbh->state =~ /^(54000|22021|XX000)$/;
+        return;
+    };
 
     if ( $fts_config->{'Table'} ne 'Attachments' ) {
         process_pg_insert( $error_handling );

commit a3e7081db3be9716563b00c139c658f9bc9d4007
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon Mar 16 13:16:21 2015 -0400

    Drop the severity of known indexing failures to "info"
    
    These failures are known failure mores that cannot easily be addressed.
    While the failure merits logging, as they are not unexpected in most
    installs, there is no direct solution and they can, in most cases, be
    ignored.  As such, "info" is a fine log level for a not-unexpected and
    unhandle-able error.

diff --git a/sbin/rt-fulltext-indexer.in b/sbin/rt-fulltext-indexer.in
index a326de7..eca1a32 100644
--- a/sbin/rt-fulltext-indexer.in
+++ b/sbin/rt-fulltext-indexer.in
@@ -248,8 +248,8 @@ sub process_mysql {
         sub {
             my ($id) = @_;
             if ($dbh->err == 1366 and $dbh->state eq "HY000") {
-                warn "Attachment $id cannot be indexed. Most probably it contains invalid UTF8 bytes. ".
-                    "Error: ". $dbh->errstr;
+                $RT::Logger->info("Attachment $id cannot be indexed. Most probably it contains invalid UTF8 bytes. ".
+                    "Error: ". $dbh->errstr);
             } else {
                 die "Attachment $id cannot be indexed: " . $dbh->errstr;
             }
@@ -263,14 +263,14 @@ sub process_pg {
         my ($id) = @_;
         my $dbh = $RT::Handle->dbh;
         if ( $dbh->err == 7 && $dbh->state eq '54000' ) {
-            warn "Attachment $id cannot be indexed. Most probably it contains too many unique words. ".
-                "Error: ". $dbh->errstr;
+            $RT::Logger->info("Attachment $id cannot be indexed. Most probably it contains too many unique words. ".
+                $dbh->errstr);
         } elsif ( $dbh->err == 7 && $dbh->state eq '22021' ) {
-            warn "Attachment $id cannot be indexed. Most probably it contains invalid UTF8 bytes. ".
-                "Error: ". $dbh->errstr;
+            $RT::Logger->info("Attachment $id cannot be indexed. Most probably it contains invalid UTF8 bytes. ".
+                $dbh->errstr);
         } elsif ( $dbh->err == 7 && $dbh->state eq 'XX000' ) {
-            warn "Attachment $id cannot be indexed; it is too large.  ".
-                "Error: ". $dbh->errstr;
+            $RT::Logger->info("Attachment $id cannot be indexed; it is too large.  ".
+                $dbh->errstr);
         } else {
             die "Attachment $id cannot be indexed: " . $dbh->errstr;
         }
diff --git a/t/fts/indexed_pg.t b/t/fts/indexed_pg.t
index 1494fde..7a085cd 100644
--- a/t/fts/indexed_pg.t
+++ b/t/fts/indexed_pg.t
@@ -106,12 +106,7 @@ $content .= "$_\n" for 1..200_000;
     { Subject => 'Long content',  Content => $content  },
     { Subject => 'More short',    Content => '50' },
 );
-
-my ($exit_code, $output) = RT::Test->run_and_capture(
-    command => $RT::SbinPath .'/rt-fulltext-indexer'
-);
-like($output, qr/string is too long for tsvector/, "Got a warning for the ticket");
-ok(!$exit_code, "set up index");
+sync_index();
 
 # The long content is skipped entirely
 run_tests(

-----------------------------------------------------------------------


More information about the rt-commit mailing list