[Rt-commit] rt branch, 4.0/base64-in-rescue-outlook, created. rt-4.0.6-266-gf405afc
Jim Brandt
jbrandt at bestpractical.com
Thu Aug 16 13:05:31 EDT 2012
The branch, 4.0/base64-in-rescue-outlook has been created
at f405afc8d064ab36185a38fd34de58e646f0ffea (commit)
- Log -----------------------------------------------------------------
commit f405afc8d064ab36185a38fd34de58e646f0ffea
Author: Jim Brandt <jbrandt at bestpractical.com>
Date: Tue May 1 10:57:24 2012 -0400
Enable Outlook newline cleanup on base64 email.
Move the RescueOutlook call to come after email is decoded.
Allow double newline cleanup on base64 email. Trigger
based on a config option and some headers that should
be MS-specific.
Email samples also have a single space on the extra lines,
so remove trailing space while cleaning up.
diff --git a/etc/RT_Config.pm.in b/etc/RT_Config.pm.in
index 192fada..0806925 100755
--- a/etc/RT_Config.pm.in
+++ b/etc/RT_Config.pm.in
@@ -459,6 +459,22 @@ Set($ExtractSubjectTagNoMatch, ( ${RT::EmailSubjectTagRegex}
? qr/\[(?:${RT::EmailSubjectTagRegex}) #\d+\]/
: qr/\[\Q$RT::rtname\E #\d+\]/));
+=item C<$CleanMailDoubleNewlines>
+
+Some email clients create a plain text version of HTML-formatted
+email to help other clients that read only plain text.
+Unfortunately, the plain text parts sometimes end up with
+doubled newlines and these can then end up in RT. This
+is most often seen in MS Outlook.
+
+Enable this option to have RT attempt to clean up double
+newlines in email from MS Outlook. Note that it may
+clean up intentional double newlines as well.
+
+=cut
+
+Set( $CleanMailDoubleNewlines, 0);
+
=back
diff --git a/lib/RT/EmailParser.pm b/lib/RT/EmailParser.pm
index 4cf4184..d3b9202 100644
--- a/lib/RT/EmailParser.pm
+++ b/lib/RT/EmailParser.pm
@@ -131,8 +131,6 @@ sub SmartParseMIMEEntityFromScalar {
}
};
- $self->RescueOutlook;
-
#If for some reason we weren't able to parse the message using a temp file
# try it with a scalar
if ( $@ || !$self->Entity ) {
@@ -568,50 +566,84 @@ return 1 if it does find the problem in the entity and get it fixed.
sub RescueOutlook {
my $self = shift;
my $mime = $self->Entity();
- return unless $mime;
-
- my $mailer = $mime->head->get('X-Mailer');
- # 12.0 is outlook 2007, 14.0 is 2010
- if ( $mailer && $mailer =~ /Microsoft(?:.*?)Outlook 1[2-4]\./ ) {
- my $text_part;
- if ( $mime->head->get('Content-Type') =~ m{multipart/mixed} ) {
- my $first = $mime->parts(0);
- if ( $first->head->get('Content-Type') =~ m{multipart/alternative} )
- {
- my $inner_first = $first->parts(0);
- if ( $inner_first->head->get('Content-Type') =~ m{text/plain} )
- {
- $text_part = $inner_first;
- }
- }
- }
- elsif ( $mime->head->get('Content-Type') =~ m{multipart/alternative} ) {
- my $first = $mime->parts(0);
- if ( $first->head->get('Content-Type') =~ m{text/plain} ) {
- $text_part = $first;
- }
- }
- if ($text_part) {
-
- # use the unencoded string
- my $content = $text_part->bodyhandle->as_string;
- if ( $content =~ s/\n\n/\n/g ) {
- # only write only if we did change the content
- if ( my $io = $text_part->open("w") ) {
- $io->print($content);
- $io->close;
- return 1;
- }
- else {
- $RT::Logger->error("can't write to body");
- }
- }
- }
+ return unless $mime && $self->LooksLikeMSEmail($mime);
+
+ my $text_part;
+ if ( $mime->head->get('Content-Type') =~ m{multipart/mixed} ) {
+ my $first = $mime->parts(0);
+ if ( $first->head->get('Content-Type') =~ m{multipart/alternative} ) {
+ my $inner_first = $first->parts(0);
+ if ( $inner_first->head->get('Content-Type') =~ m{text/plain} ) {
+ $text_part = $inner_first;
+ }
+ }
+ } elsif ( $mime->head->get('Content-Type') =~ m{multipart/alternative} ) {
+ my $first = $mime->parts(0);
+ if ( $first->head->get('Content-Type') =~ m{text/plain} ) {
+ $text_part = $first;
+ }
+ }
+
+ # Add base64 since we've seen examples of double newlines with
+ # this type too. Need an example of a multi-part base64 to
+ # handle that permutation if it exists.
+ elsif ( $mime->head->get('Content-Transfer-Encoding') =~ m{base64} ) {
+ $text_part = $mime; # Assuming single part, already decoded.
}
+
+ if ($text_part) {
+ # use the unencoded string
+ my $content = $text_part->bodyhandle->as_string;
+ if ( $content =~ s/\n\n/\n/g ) {
+ # Outlook puts a space on extra newlines, remove it
+ $content =~ s/\ +$//mg;
+
+ # only write only if we did change the content
+ if ( my $io = $text_part->open("w") ) {
+ $io->print($content);
+ $io->close;
+ $RT::Logger->debug("Removed extra newlines from MS Outlook message.");
+ return 1;
+ } else {
+ $RT::Logger->error("Can't write to body to fix newlines");
+ }
+ }
+ }
+
return;
}
+=head1 LooksLikeMSEmail
+
+Try to determine if the current email may have
+come from MS Outlook or gone through Exchange, and therefore
+may have extra newlines added.
+
+=cut
+
+sub LooksLikeMSEmail {
+ my $self = shift;
+ my $mime = shift;
+
+ my $mailer = $mime->head->get('X-Mailer');
+
+ # 12.0 is outlook 2007, 14.0 is 2010
+ return 1 if ( $mailer && $mailer =~ /Microsoft(?:.*?)Outlook 1[2-4]\./ );
+
+ if ( RT->Config->Get('CleanMailDoubleNewlines') ){
+
+ # Check for some headers that might
+ # indicate this came from Outlook or through Exchange.
+ # A sample we received had the headers X-MS-Has-Attach: and
+ # X-MS-Tnef-Correlator: and both had no value.
+
+ my @tags = $mime->head->tags();
+ return 1 if grep { /^X-MS-/ } @tags;
+ }
+
+ return 0; # Doesn't look like MS email.
+}
sub DESTROY {
my $self = shift;
diff --git a/lib/RT/Interface/Email.pm b/lib/RT/Interface/Email.pm
index 2594e99..e899038 100644
--- a/lib/RT/Interface/Email.pm
+++ b/lib/RT/Interface/Email.pm
@@ -1404,6 +1404,7 @@ sub Gateway {
}
@mail_plugins = grep !$skip_plugin{"$_"}, @mail_plugins;
$parser->_DecodeBodies;
+ $parser->RescueOutlook;
$parser->_PostProcessNewEntity;
my $head = $Message->head;
diff --git a/t/mail/outlook.t b/t/mail/outlook.t
index 00bbc94..6d851e5 100644
--- a/t/mail/outlook.t
+++ b/t/mail/outlook.t
@@ -1,41 +1,41 @@
#!/usr/bin/perl -w
# BEGIN BPS TAGGED BLOCK {{{
-#
+#
# COPYRIGHT:
-#
-# This software is Copyright (c) 1996-2004 Best Practical Solutions, LLC
+#
+# This software is Copyright (c) 1996-2004 Best Practical Solutions, LLC
# <jesse.com>
-#
+#
# (Except where explicitly superseded by other copyright notices)
-#
-#
+#
+#
# LICENSE:
-#
+#
# This work is made available to you under the terms of Version 2 of
# the GNU General Public License. A copy of that license should have
# been provided with this software, but in any event can be snarfed
# from www.gnu.org.
-#
+#
# This work is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
-#
+#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 or visit their web page on the internet at
# http://www.gnu.org/copyleft/gpl.html.
-#
-#
+#
+#
# CONTRIBUTION SUBMISSION POLICY:
-#
+#
# (The following paragraph is not intended to limit the rights granted
# to you to modify and distribute this software under the terms of
# the GNU General Public License and is only of importance to you if
# you choose to contribute your changes and enhancements to the
# community by submitting them to Best Practical Solutions, LLC.)
-#
+#
# By intentionally submitting any modifications, corrections or
# derivatives to this work, or any other work intended for use with
# Request Tracker, to Best Practical Solutions, LLC, you confirm that
@@ -44,19 +44,25 @@
# royalty-free, perpetual, license to use, copy, create derivative
# works based on those contributions, and sublicense and distribute
# those contributions and any derivatives thereof.
-#
+#
# END BPS TAGGED BLOCK }}}
=head1 NAME
-rt-mailgate - Mail interface to RT3.
+outlook.t
+
+=head1 DESCRIPTION
+
+Tests for custom routines to work with email from Outlook.
=cut
use strict;
use warnings;
-use RT::Test tests => 42;
+use RT::Test tests => 66;
+
+RT->Config->Set('CleanMailDoubleNewlines', 1);
# 12.0 is outlook 2007, 14.0 is 2010
for my $mailer ( 'Microsoft Office Outlook 12.0', 'Microsoft Outlook 14.0' ) {
@@ -199,8 +205,75 @@ EOF
test_email( $text, $content,
$mailer . ' with only text/plain, \n\n are not replaced' );
}
+
+ diag "Test mail with with outlook, content type is base64";
+ {
+ my $text = <<EOF;
+From: root\@localhost
+X-Mailer: $mailer
+To: rt\@@{[RT->Config->Get('rtname')]}
+Subject: outlook basic test
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: base64
+
+VGhpcyBpcyB0aGUgYm9keSBvZiBhbiBlbWFpbC4KCkl0IGhhcyBtdWx0aXBs
+ZSBleHRyYSBuZXdsaW5lcy4KCgoKTGlrZSBhIG1hbmdsZWQgT3V0bG9vayBt
+ZXNzYWdlIG1pZ2h0LgoKCgpKb2huIFNtaXRoCgpTb21lIENvbXBhbnkKCmVt
+YWlsQHNvbWVjby5jb20KCg==
+EOF
+
+ my $content = <<EOF;
+This is the body of an email.
+It has multiple extra newlines.
+
+Like a mangled Outlook message might.
+
+John Smith
+Some Company
+email\@someco.com
+EOF
+ test_email( $text, $content,
+ $mailer . ' with base64, \n\n are replaced' );
+ }
+}
+
+# In a sample we received, the two X-MS- headers included
+# below were both present and had no values. For now, using
+# the existence of these headers as evidence of MS Outlook
+# or Exchange.
+
+diag "Test mail with with outlook, no X-Mailer, content type is base64";
+{
+ my $text = <<EOF;
+From: root\@localhost
+To: rt\@@{[RT->Config->Get('rtname')]}
+Subject: outlook basic test
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: base64
+X-MS-Has-Attach:
+X-MS-Tnef-Correlator:
+
+VGhpcyBpcyB0aGUgYm9keSBvZiBhbiBlbWFpbC4KCkl0IGhhcyBtdWx0aXBs
+ZSBleHRyYSBuZXdsaW5lcy4KCgoKTGlrZSBhIG1hbmdsZWQgT3V0bG9vayBt
+ZXNzYWdlIG1pZ2h0LgoKCgpKb2huIFNtaXRoCgpTb21lIENvbXBhbnkKCmVt
+YWlsQHNvbWVjby5jb20KCg==
+EOF
+
+ my $content = <<EOF;
+This is the body of an email.
+It has multiple extra newlines.
+
+Like a mangled Outlook message might.
+
+John Smith
+Some Company
+email\@someco.com
+EOF
+ test_email( $text, $content,
+ ' with base64, no X-Mailer, \n\n are replaced' );
}
+
diag "Test mail with with multipart/alternative but x-mailer is not outlook ";
{
my $text = <<EOF;
@@ -234,7 +307,6 @@ Content-Transfer-Encoding: quoted-printable
------=_NextPart_000_0004_01CB045C.A5A075D0--
-
EOF
my $content = <<EOF;
@@ -250,6 +322,129 @@ EOF
test_email( $text, $content, 'without outlook, \n\n are not replaced' );
}
+diag "Sample multipart email with Exchange headers";
+{
+ my $text = <<EOF;
+X-MimeOLE: Produced By Microsoft Exchange V6.5
+Received: by example.com
+ id <01CD63FC.33F4C15C\@example.com>; Tue, 17 Jul 2012 10:11:51 +0100
+MIME-Version: 1.0
+Content-Type: multipart/alternative;
+ boundary="----_=_NextPart_001_01CD63FC.33F4C15C"
+Content-class: urn:content-classes:message
+Subject: outlook basic test
+Date: Tue, 17 Jul 2012 10:11:50 +0100
+Message-ID: <AA6CEAFB02FF244999046B2A6B6B9D6F05FF9D12\@example.com>
+X-MS-Has-Attach:
+X-MS-TNEF-Correlator:
+Thread-Topic: Testing Outlook HTML
+Thread-Index: Ac1j/DNs7ly963bnRt63SJw9DkGwyw==
+From: root\@localhost
+To: rt\@@{[RT->Config->Get('rtname')]}
+
+This is a multi-part message in MIME format.
+
+------_=_NextPart_001_01CD63FC.33F4C15C
+Content-Type: text/plain;
+ charset="us-ascii"
+Content-Transfer-Encoding: quoted-printable
+
+This email contains a line of text containing multiple sentences. Where
+will RT wrap this when the text is quoted? What about the footer below?
+
+=20
+
+This is a different line, with a blank line (paragraph) above. Will
+there be additional blank lines when the text is quoted?
+
+=20
+
+This isthesig
+
+=20
+
+
+------_=_NextPart_001_01CD63FC.33F4C15C
+Content-Type: text/html;
+ charset="us-ascii"
+Content-Transfer-Encoding: quoted-printable
+
+<html xmlns:v=3D"urn:schemas-microsoft-com:vml" =
+xmlns:o=3D"urn:schemas-microsoft-com:office:office" =
+xmlns:w=3D"urn:schemas-microsoft-com:office:word" =
+xmlns:m=3D"http://schemas.microsoft.com/office/2004/12/omml" =
+xmlns=3D"http://www.w3.org/TR/REC-html40"><head><META =
+HTTP-EQUIV=3D"Content-Type" CONTENT=3D"text/html; =
+charset=3Dus-ascii"><meta name=3DGenerator content=3D"Microsoft Word 12 =
+(filtered medium)"><style><!--
+/* Font Definitions */
+\@font-face
+ {font-family:"Cambria Math";
+ panose-1:2 4 5 3 5 4 6 3 2 4;}
+\@font-face
+ {font-family:Calibri;
+ panose-1:2 15 5 2 2 2 4 3 2 4;}
+/* Style Definitions */
+p.MsoNormal, li.MsoNormal, div.MsoNormal
+ {margin:0in;
+ margin-bottom:.0001pt;
+ font-size:11.0pt;
+ font-family:"Calibri","sans-serif";}
+a:link, span.MsoHyperlink
+ {mso-style-priority:99;
+ color:blue;
+ text-decoration:underline;}
+a:visited, span.MsoHyperlinkFollowed
+ {mso-style-priority:99;
+ color:purple;
+ text-decoration:underline;}
+span.EmailStyle17
+ {mso-style-type:personal-compose;
+ font-family:"Calibri","sans-serif";
+ color:windowtext;}
+.MsoChpDefault
+ {mso-style-type:export-only;}
+\@page WordSection1
+ {size:8.5in 11.0in;
+ margin:1.0in 1.0in 1.0in 1.0in;}
+div.WordSection1
+ {page:WordSection1;}
+--></style><!--[if gte mso 9]><xml>
+<o:shapedefaults v:ext=3D"edit" spidmax=3D"1026" />
+</xml><![endif]--><!--[if gte mso 9]><xml>
+<o:shapelayout v:ext=3D"edit">
+<o:idmap v:ext=3D"edit" data=3D"1" />
+</o:shapelayout></xml><![endif]--></head><body lang=3DEN-US link=3Dblue =
+vlink=3Dpurple><div class=3DWordSection1><p class=3DMsoNormal>This email =
+contains a line of text containing multiple sentences. Where will =
+RT wrap this when the text is quoted? What about the footer =
+below?<o:p></o:p></p><p class=3DMsoNormal><o:p> </o:p></p><p =
+class=3DMsoNormal>This is a different line, with a blank line =
+(paragraph) above. Will there be additional blank lines when the =
+text is quoted?<o:p></o:p></p><p =
+class=3DMsoNormal><o:p> </o:p></p><p class=3DMsoNormal><span =
+lang=3DEN-GB =
+style=3D'font-size:7.5pt;font-family:"Arial","sans-serif"'>This isthesig =
+</span><o:p></o:p></p><p =
+class=3DMsoNormal><o:p> </o:p></p></div></body></html>
+------_=_NextPart_001_01CD63FC.33F4C15C--
+EOF
+
+ my $content = <<EOF;
+This email contains a line of text containing multiple sentences. Where
+will RT wrap this when the text is quoted? What about the footer below?
+
+This is a different line, with a blank line (paragraph) above. Will
+there be additional blank lines when the text is quoted?
+
+This isthesig
+
+EOF
+
+ test_email( $text, $content,
+ 'Another sample multipart message with Exchange headers' );
+}
+
sub test_email {
my ( $text, $content, $msg ) = @_;
my ( $status, $id ) = RT::Test->send_via_mailgate($text);
-----------------------------------------------------------------------
More information about the Rt-commit
mailing list