[Rt-commit] rt branch, 4.4/munge-more-attachments, created. rt-4.4.3-198-ga342442b3

? sunnavy sunnavy at bestpractical.com
Wed Jan 23 15:36:48 EST 2019


The branch, 4.4/munge-more-attachments has been created
        at  a342442b329311d1cc82179aa973fc31f9758312 (commit)

- Log -----------------------------------------------------------------
commit 67d87737c9924065a9a1c2e64aef526c56eeaeed
Author: sunnavy <sunnavy at bestpractical.com>
Date:   Thu Jan 24 01:22:24 2019 +0800

    Abstract method to return the list of mime types that indicate email messages

diff --git a/lib/RT/Util.pm b/lib/RT/Util.pm
index 3538c00aa..aaa9d1530 100644
--- a/lib/RT/Util.pm
+++ b/lib/RT/Util.pm
@@ -52,7 +52,7 @@ use warnings;
 
 
 use base 'Exporter';
-our @EXPORT = qw/safe_run_child mime_recommended_filename EntityLooksLikeEmailMessage/;
+our @EXPORT = qw/safe_run_child mime_recommended_filename EntityLooksLikeEmailMessage EmailContentTypes/;
 
 use Encode qw/encode/;
 
@@ -230,16 +230,26 @@ sub EntityLooksLikeEmailMessage {
     # MIME::Parser used.
     my $mime_type = $entity->mime_type();
 
-    # This is the same list of MIME types MIME::Parser uses. The partial and
-    # external-body types are unlikely to produce usable attachments, but they
-    # are still recognized as email for the purposes of this function.
-
-    my @email_types = ('message/rfc822', 'message/partial', 'message/external-body');
+    my @email_types = EmailContentTypes();
 
     return 1 if grep { $mime_type eq $_ } @email_types;
     return 0;
 }
 
+=head2 EmailContentTypes
+
+Return MIME types that indicate email messages.
+
+=cut
+
+sub EmailContentTypes {
+
+    # This is the same list of MIME types MIME::Parser uses. The partial and
+    # external-body types are unlikely to produce usable attachments, but they
+    # are still recognized as email for the purposes of this function.
+    return ( 'message/rfc822', 'message/partial', 'message/external-body' );
+}
+
 RT::Base->_ImportOverlays();
 
 1;

commit 0d603e83f1fb65f0195313fd9a4c66ac5e45798e
Author: sunnavy <sunnavy at bestpractical.com>
Date:   Thu Jan 24 01:30:09 2019 +0800

    Refactor ReplaceAttachments method mainly to cover more attachments
    
    Previously we only replaced attachments of "text/plain" and "text/html",
    which is good enough for "Content" replacement but sadly not for
    "Headers", as we missed quite common types like "multipart/mixed" and
    more. This commits imporves "Headers" replacement by covering all the
    types of "multiplart/*" and also embeded email ones like
    "message/rfc822", 'message/partial' and 'message/external-body'.
    
    Code is also refactored to create Munge txns as soon as possible for
    better consistency, and the memory usage is also reduced as we don't
    need to keep all the ticket objects.

diff --git a/lib/RT/Attachments.pm b/lib/RT/Attachments.pm
index 474a369d8..fbbf3fc70 100644
--- a/lib/RT/Attachments.pm
+++ b/lib/RT/Attachments.pm
@@ -271,60 +271,101 @@ sub ReplaceAttachments {
 
     return ( 0, $self->loc('Provide a search string to search on') ) unless $args{Search};
 
-    $self->Limit(
-        ENTRYAGGREGATOR => 'OR',
-        FIELD           => 'Headers',
-        OPERATOR        => 'LIKE',
-        VALUE           => $args{Search},
-        SUBCLAUSE       => 'Attachments',
-    ) if $args{Headers};
-    $self->Limit(
-        ENTRYAGGREGATOR => 'OR',
-        FIELD           => 'Content',
-        OPERATOR        => 'LIKE',
-        VALUE           => $args{Search},
-        SUBCLAUSE       => 'Attachments',
-    ) if $args{Content};
-    $self->Limit(
-        FIELD           => 'ContentType',
-        OPERATOR        => 'IN',
-        VALUE           => ['text/plain', 'text/html'],
+
+    my %munged;
+    my $create_munge_txn = sub {
+        my $ticket = shift;
+        if ( !$munged{ $ticket->id } ) {
+            my ( $ret, $msg ) = $ticket->_NewTransaction( Type => "Munge" );
+            if ($ret) {
+                $munged{ $ticket->id } = 1;
+            }
+            else {
+                RT::Logger->error($msg);
+            }
+        }
+    };
+
+    my $attachments = $self->Clone;
+    $attachments->Limit(
+        FIELD     => 'ContentEncoding',
+        VALUE     => 'none',
+        SUBCLAUSE => 'Encoding',
     );
-    $self->Limit(
+    $attachments->Limit(
         FIELD           => 'ContentEncoding',
-        VALUE           => 'none',
+        OPERATOR        => 'IS',
+        VALUE           => 'NULL',
+        SUBCLAUSE       => 'Encoding',
     );
 
-    my %tickets;
-    my ($ret, $msg);
-    while (my $attachment = $self->Next) {
-        my $content_replaced;
-        if ( $args{Headers} ) {
-            ($ret, $msg) = $attachment->ReplaceHeaders(Search => $args{Search}, Replacement => $args{Replacement});
-            $content_replaced ||= $ret;
+    if ( $args{Headers} ) {
+        my $atts = $attachments->Clone;
+        $atts->Limit(
+            FIELD    => 'Headers',
+            OPERATOR => 'LIKE',
+            VALUE    => $args{Search},
+        );
+        $atts->Limit(
+            FIELD     => 'ContentType',
+            OPERATOR  => 'IN',
+            VALUE     => [ RT::Util::EmailContentTypes(), 'text/plain', 'text/html' ],
+            SUBCLAUSE => 'Types',
+        );
+        $atts->Limit(
+            FIELD           => 'ContentType',
+            OPERATOR        => 'STARTSWITH',
+            VALUE           => 'multipart/',
+            SUBCLAUSE       => 'Types',
+            ENTRYAGGREGATOR => 'OR',
+        );
 
-            RT::Logger->error($msg) unless $ret;
+        while ( my $att = $atts->Next ) {
+            my ( $ret, $msg ) = $att->ReplaceHeaders(
+                Search      => $args{Search},
+                Replacement => $args{Replacement},
+            );
+
+            if ( $ret ) {
+                $create_munge_txn->( $att->TransactionObj->TicketObj );
+            }
+            else {
+                RT::Logger->error($msg);
+            }
         }
+    }
 
-        if ( $args{Content} ) {
-            ($ret, $msg) = $attachment->ReplaceContent(Search => $args{Search}, Replacement => $args{Replacement});
-            $content_replaced ||= $ret;
+    if ( $args{Content} ) {
+        my $atts = $attachments->Clone;
+        $atts->Limit(
+            FIELD     => 'Content',
+            OPERATOR  => 'LIKE',
+            VALUE     => $args{Search},
+            SUBCLAUSE => 'Content',
+        );
+        $atts->Limit(
+            FIELD    => 'ContentType',
+            OPERATOR => 'IN',
+            VALUE    => [ 'text/plain', 'text/html' ],
+        );
 
-            RT::Logger->error($msg) unless $ret;
+        while ( my $att = $atts->Next ) {
+            my ( $ret, $msg ) = $att->ReplaceContent(
+                Search      => $args{Search},
+                Replacement => $args{Replacement},
+            );
+
+            if ( $ret ) {
+                $create_munge_txn->( $att->TransactionObj->TicketObj );
+            }
+            else {
+                RT::Logger->error($msg);
+            }
         }
-
-        my $ticket = $attachment->TransactionObj->TicketObj;
-        $tickets{$ticket->Id} ||= $ticket if $content_replaced;
     }
 
-    foreach my $id ( sort { $a <=> $b } keys %tickets ) {
-        (my $transaction, $msg, my $trans) = $tickets{$id}->_NewTransaction (
-            Type     => "Munge",
-        );
-        RT::Logger->error($msg) unless $transaction;
-    }
-    my $count = scalar keys %tickets;
-    return ( 1, $self->loc( "Updated [_1] ticket's attachment content", $count ) );
+    my $count = scalar keys %munged;
+    return ( 1, $self->loc( "Updated [quant,_1,ticket's,tickets'] attachment content", $count ) );
 }
 
 RT::Base->_ImportOverlays();

commit a342442b329311d1cc82179aa973fc31f9758312
Author: sunnavy <sunnavy at bestpractical.com>
Date:   Thu Jan 24 03:29:47 2019 +0800

    Munge quoted-printable attachments if the searched string keeps intact in QP

diff --git a/lib/RT/Attachments.pm b/lib/RT/Attachments.pm
index fbbf3fc70..b385e9962 100644
--- a/lib/RT/Attachments.pm
+++ b/lib/RT/Attachments.pm
@@ -299,6 +299,15 @@ sub ReplaceAttachments {
         SUBCLAUSE       => 'Encoding',
     );
 
+    # Adding "\n" is to avoid trailing "=" in QP encoding
+    if ( MIME::QuotedPrint::encode("$args{Search}\n") eq "$args{Search}\n" ) {
+        $attachments->Limit(
+            FIELD     => 'ContentEncoding',
+            VALUE     => 'quoted-printable',
+            SUBCLAUSE => 'Encoding',
+        );
+    }
+
     if ( $args{Headers} ) {
         my $atts = $attachments->Clone;
         $atts->Limit(

-----------------------------------------------------------------------


More information about the rt-commit mailing list