[Rt-commit] rt branch, 4.0/utf8-reckoning, created. rt-4.0.21-54-ge96002a

Alex Vandiver alexmv at bestpractical.com
Wed Sep 3 13:49:16 EDT 2014


The branch, 4.0/utf8-reckoning has been created
        at  e96002ae4dadf2125e3ead0e1940cb5df9f4c78a (commit)

- Log -----------------------------------------------------------------
commit e93c82e27828d55e1ffcd09728f26cfedce999e2
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon May 12 16:48:32 2014 -0400

    Re-indent _EncodeLOB and _DecodeLOB

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 038865d..4992d1c 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -747,72 +747,71 @@ evaluate and encode it.  It will return an octet string.
 =cut
 
 sub _EncodeLOB {
-        my $self = shift;
-        my $Body = shift;
-        my $MIMEType = shift || '';
-        my $Filename = shift;
+    my $self = shift;
+    my $Body = shift;
+    my $MIMEType = shift || '';
+    my $Filename = shift;
 
-        my $ContentEncoding = 'none';
+    my $ContentEncoding = 'none';
 
-        #get the max attachment length from RT
-        my $MaxSize = RT->Config->Get('MaxAttachmentSize');
+    #get the max attachment length from RT
+    my $MaxSize = RT->Config->Get('MaxAttachmentSize');
 
-        #if the current attachment contains nulls and the
-        #database doesn't support embedded nulls
+    #if the current attachment contains nulls and the
+    #database doesn't support embedded nulls
 
-        if ( ( !$RT::Handle->BinarySafeBLOBs ) && ( $Body =~ /\x00/ ) ) {
+    if ( ( !$RT::Handle->BinarySafeBLOBs ) && ( $Body =~ /\x00/ ) ) {
 
-            # set a flag telling us to mimencode the attachment
-            $ContentEncoding = 'base64';
+        # set a flag telling us to mimencode the attachment
+        $ContentEncoding = 'base64';
 
-            #cut the max attchment size by 25% (for mime-encoding overhead.
-            $RT::Logger->debug("Max size is $MaxSize");
-            $MaxSize = $MaxSize * 3 / 4;
-        # Some databases (postgres) can't handle non-utf8 data
-        } elsif (    !$RT::Handle->BinarySafeBLOBs
-                  && $Body =~ /\P{ASCII}/
-                  && !Encode::is_utf8( $Body, 1 ) ) {
-              $ContentEncoding = 'quoted-printable';
-        }
+        #cut the max attchment size by 25% (for mime-encoding overhead.
+        $RT::Logger->debug("Max size is $MaxSize");
+        $MaxSize = $MaxSize * 3 / 4;
+    # Some databases (postgres) can't handle non-utf8 data
+    } elsif (    !$RT::Handle->BinarySafeBLOBs
+              && $Body =~ /\P{ASCII}/
+              && !Encode::is_utf8( $Body, 1 ) ) {
+          $ContentEncoding = 'quoted-printable';
+    }
 
-        #if the attachment is larger than the maximum size
-        if ( ($MaxSize) and ( $MaxSize < length($Body) ) ) {
+    #if the attachment is larger than the maximum size
+    if ( ($MaxSize) and ( $MaxSize < length($Body) ) ) {
 
-            # if we're supposed to truncate large attachments
-            if (RT->Config->Get('TruncateLongAttachments')) {
+        # if we're supposed to truncate large attachments
+        if (RT->Config->Get('TruncateLongAttachments')) {
 
-                # truncate the attachment to that length.
-                $Body = substr( $Body, 0, $MaxSize );
+            # truncate the attachment to that length.
+            $Body = substr( $Body, 0, $MaxSize );
 
-            }
+        }
 
-            # elsif we're supposed to drop large attachments on the floor,
-            elsif (RT->Config->Get('DropLongAttachments')) {
+        # elsif we're supposed to drop large attachments on the floor,
+        elsif (RT->Config->Get('DropLongAttachments')) {
 
-                # drop the attachment on the floor
-                $RT::Logger->info( "$self: Dropped an attachment of size "
-                                   . length($Body));
-                $RT::Logger->info( "It started: " . substr( $Body, 0, 60 ) );
-                $Filename .= ".txt" if $Filename;
-                return ("none", "Large attachment dropped", "text/plain", $Filename );
-            }
+            # drop the attachment on the floor
+            $RT::Logger->info( "$self: Dropped an attachment of size "
+                               . length($Body));
+            $RT::Logger->info( "It started: " . substr( $Body, 0, 60 ) );
+            $Filename .= ".txt" if $Filename;
+            return ("none", "Large attachment dropped", "text/plain", $Filename );
         }
+    }
 
-        # if we need to mimencode the attachment
-        if ( $ContentEncoding eq 'base64' ) {
+    # if we need to mimencode the attachment
+    if ( $ContentEncoding eq 'base64' ) {
 
-            # base64 encode the attachment
-            Encode::_utf8_off($Body);
-            $Body = MIME::Base64::encode_base64($Body);
-
-        } elsif ($ContentEncoding eq 'quoted-printable') {
-            Encode::_utf8_off($Body);
-            $Body = MIME::QuotedPrint::encode($Body);
-        }
+        # base64 encode the attachment
+        Encode::_utf8_off($Body);
+        $Body = MIME::Base64::encode_base64($Body);
 
+    } elsif ($ContentEncoding eq 'quoted-printable') {
+        Encode::_utf8_off($Body);
+        $Body = MIME::QuotedPrint::encode($Body);
+    }
 
-        return ($ContentEncoding, $Body, $MIMEType, $Filename );
 
+    return ($ContentEncoding, $Body, $MIMEType, $Filename );
 }
 
 =head2 _DecodeLOB
@@ -852,9 +851,9 @@ sub _DecodeLOB {
         return ( $self->loc( "Unknown ContentEncoding [_1]", $ContentEncoding ) );
     }
     if ( RT::I18N::IsTextualContentType($ContentType) ) {
-       $Content = Encode::decode('UTF-8',$Content,Encode::FB_PERLQQ) unless Encode::is_utf8($Content);
+        $Content = Encode::decode('UTF-8',$Content,Encode::FB_PERLQQ) unless Encode::is_utf8($Content);
     }
-        return ($Content);
+    return ($Content);
 }
 
 # A helper table for links mapping to make it easier

commit 802dc8ddff71c1dd0f8c14b23dc874aae7951b73
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Mon May 12 18:04:19 2014 -0400

    Respect the database Content-Type header in decoding textual parts
    
    The definition of "texual" data has changed over time.  Specifically,
    7365c08 caused text/html messages to begin being stored as utf-8 in the
    database; prior to that, the claimed "charset" and bytes in the body
    were left unmolested during insertion.
    
    text/html attachments inserted into the database prior to 7365c08,
    however, are now expected to be utf-8 when being extracted from the
    database.  This causes PERLQQ'd garbage to be displayed for the non-UTF8
    content stored in the database.  This type of error is likely to also
    re-occur in the future whenever the definition of "textual" data
    (i.e. data we transcode on insertion) changes.
    
    Respect the Content-Type header when decoding data from the database, or
    guess its value from the body; this mirrors the logic in
    RT::I18N::SetMIMEEntityToEncoding, which is what is done for
    currently-detected-as-textual parts on insert.  In cases like text/html
    prior to 7365c08, the Content-Type header was not altered during
    database insertion -- and at worst, the claimed character set is
    incorrect and decoding will result in PERLQQ'd garbage.  This is no
    worse than said message were detected, received, converted, and stored
    in the database as text.

diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index 07fdea3..9bfe588 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -289,7 +289,7 @@ before returning it.
 sub Content {
     my $self = shift;
     return $self->_DecodeLOB(
-        $self->ContentType,
+        $self->GetHeader('Content-Type'),  # Includes charset, unlike ->ContentType
         $self->ContentEncoding,
         $self->_Value('Content', decode_utf8 => 0),
     );
diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 4992d1c..8ee0ff8 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -814,7 +814,7 @@ sub _EncodeLOB {
     return ($ContentEncoding, $Body, $MIMEType, $Filename );
 }
 
-=head2 _DecodeLOB
+=head2 _DecodeLOB C<ContentType>, C<ContentEncoding>, C<Content>
 
 Unpacks data stored in the database, which may be base64 or QP encoded
 because of our need to store binary and badly encoded data in columns
@@ -830,6 +830,12 @@ This is similar to how we filter all data coming in via the web UI in
 RT::Interface::Web::DecodeARGS. This filter should only end up being
 applied to old data from less UTF-8-safe versions of RT.
 
+If the passed C<ContentType> includes a character set, that will be used
+to decode textual data; the default character set is UTF-8.  This is
+necessary because while we attempt to store textual data as UTF-8, the
+definition of "textual" has migrated over time, and thus we may now need
+to attempt to decode data that was previously not trancoded on insertion.
+
 Important Note - This function expects an octet string and returns a
 character string for non-binary data.
 
@@ -851,7 +857,13 @@ sub _DecodeLOB {
         return ( $self->loc( "Unknown ContentEncoding [_1]", $ContentEncoding ) );
     }
     if ( RT::I18N::IsTextualContentType($ContentType) ) {
-        $Content = Encode::decode('UTF-8',$Content,Encode::FB_PERLQQ) unless Encode::is_utf8($Content);
+        my $entity = MIME::Entity->new();
+        $entity->head->add("Content-Type", $ContentType);
+        $entity->bodyhandle( MIME::Body::Scalar->new( $Content ) );
+        my $charset = RT::I18N::_FindOrGuessCharset($entity);
+        $charset = 'utf-8' if not $charset or not Encode::find_encoding($charset);
+
+        $Content = Encode::decode($charset,$Content,Encode::FB_PERLQQ) unless Encode::is_utf8($Content);
     }
     return ($Content);
 }

commit 968c25cc173b292994f0e119f0e3253142728564
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue May 13 12:15:20 2014 -0400

    Stop needlessly frobbing utf8 internals
    
    bf87c784 swapped in this explicit _utf8_off for what had been a
    _utf8_on.  As the values are fetched with decode_utf8 => 0, $content
    contains bytes, not characters.  Explicitly turning of fthe utf8 bit is
    unnecessary, as they are already bytes.

diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index 9bfe588..b872651 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -333,10 +333,7 @@ sub OriginalContent {
         return( $self->loc("Unknown ContentEncoding [_1]", $self->ContentEncoding));
     }
 
-    # Turn *off* the SvUTF8 bits here so decode_utf8 and from_to below can work.
     local $@;
-    Encode::_utf8_off($content);
-
     if (!$enc || $enc eq '' ||  $enc eq 'utf8' || $enc eq 'utf-8') {
         # If we somehow fail to do the decode, at least push out the raw bits
         eval { return( Encode::decode_utf8($content)) } || return ($content);

commit 04b5caf77031f2b5ac416602fd2b24475f1e3aee
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue May 13 12:20:38 2014 -0400

    Decoding content, and returning characters, is incorrect
    
    The whole purpose of this function is to return bytes, not characters.
    Decoding the bytes to characters as utf8 defeats the purpose, and only
    works because RT's internal representation of characters happens to be
    utf8, which gets used by MIME::Entity to later encode the characters.
    
    Switch to simply returning the bytes, as-is.

diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index b872651..2a5ce0f 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -333,13 +333,12 @@ sub OriginalContent {
         return( $self->loc("Unknown ContentEncoding [_1]", $self->ContentEncoding));
     }
 
-    local $@;
     if (!$enc || $enc eq '' ||  $enc eq 'utf8' || $enc eq 'utf-8') {
-        # If we somehow fail to do the decode, at least push out the raw bits
-        eval { return( Encode::decode_utf8($content)) } || return ($content);
+        return ($content);
     }
 
-    eval { Encode::from_to($content, 'utf8' => $enc) } if $enc;
+    local $@;
+    eval { Encode::from_to($content, 'utf8' => $enc) };
     if ($@) {
         $RT::Logger->error("Could not convert attachment from assumed utf8 to '$enc' :".$@);
     }

commit 35eb3bb567731134e76a26380ffa8a8c7c67741b
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Tue May 13 12:23:34 2014 -0400

    Stop assuming the data in the database is utf8
    
    As noted in 802dc8d, not all content we currently call "textual" was
    always treated as such.  When re-encoding, do not assume that the
    encoding in the database is UTF-8 -- rather, read the Content-Type
    header, and examine the charset stated there.  Convert from that to the
    original encoding.

diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index 2a5ce0f..01844a7 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -320,7 +320,6 @@ sub OriginalContent {
     }
 
     return $self->Content unless RT::I18N::IsTextualContentType($self->ContentType);
-    my $enc = $self->OriginalEncoding;
 
     my $content;
     if ( !$self->ContentEncoding || $self->ContentEncoding eq 'none' ) {
@@ -333,14 +332,20 @@ sub OriginalContent {
         return( $self->loc("Unknown ContentEncoding [_1]", $self->ContentEncoding));
     }
 
-    if (!$enc || $enc eq '' ||  $enc eq 'utf8' || $enc eq 'utf-8') {
-        return ($content);
-    }
+    my $entity = MIME::Entity->new();
+    $entity->head->add("Content-Type", $self->GetHeader("Content-Type"));
+    $entity->bodyhandle( MIME::Body::Scalar->new( $content ) );
+    my $from = RT::I18N::_FindOrGuessCharset($entity);
+    $from = 'utf-8' if not $from or not Encode::find_encoding($from);
+
+    my $to = RT::I18N::_CanonicalizeCharset(
+        $self->OriginalEncoding || 'utf-8'
+    );
 
     local $@;
-    eval { Encode::from_to($content, 'utf8' => $enc) };
+    eval { Encode::from_to($content, $from => $to) };
     if ($@) {
-        $RT::Logger->error("Could not convert attachment from assumed utf8 to '$enc' :".$@);
+        $RT::Logger->error("Could not convert attachment from $from to $to: ".$@);
     }
     return $content;
 }

commit 1945c00b4febdeab68ff4447bb5d056ef7f14004
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Wed Aug 6 14:47:56 2014 -0400

    Modernize and condense t/mail/sendmail.t
    
    t/data/emails/text-html-in-russian was removed because the original
    purpose of the test was removed in 46fd04d9, after 90f9c190 stopped
    attaching text/html incoming mail to autoreplies.

diff --git a/t/data/emails/text-html-in-russian b/t/data/emails/text-html-in-russian
deleted file mode 100644
index b965b1b..0000000
--- a/t/data/emails/text-html-in-russian
+++ /dev/null
@@ -1,87 +0,0 @@
-From rickt at other-example.com  Tue Jun 17 20:39:13 2003
-Return-Path: <rickt at other-example.com>
-X-Original-To: info
-Delivered-To: mitya at vh.example.com
-Received: from example.com (mx.example.com [194.87.0.32])
-	by vh.example.com (Postfix) with ESMTP id 8D77B16E6BD
-	for <info>; Tue, 17 Jun 2003 20:39:05 +0400 (MSD)
-Received: from hotline at example.com
-  by example.com (CommuniGate Pro GROUP 4.1b7/D)
-  with GROUP id 76033026; Tue, 17 Jun 2003 20:38:00 +0400
-Received: by example.com (CommuniGate Pro PIPE 4.1b7/D)
-  with PIPE id 76033052; Tue, 17 Jun 2003 20:38:00 +0400
-Received: from [217.132.49.75] (HELO compuserve.com)
-  by example.com (CommuniGate Pro SMTP 4.1b7/D)
-  with SMTP id 76032971 for info at example.com; Tue, 17 Jun 2003 20:37:41 +0400
-Date: Wed, 18 Jun 2003 01:41:01 +0000
-From: Ó÷åáíûé Öåíòð <rickt at other-example.com>
-Subject: Ïðèãëàøàåì ðóêîâîäèòåëÿ, íà÷àëüíèêîâ ïîäðàçäåëåíèé íà òðåíèíã                                                         YXLWLJ3LPT9UHuLyGTzyuKQc06eIZ96Y6RVTCZFt
-To: Info <info at example.com>
-References: <0ID97EGL951H1907 at example.com>
-In-Reply-To: <0ID97EGL951H1907 at example.com>
-Message-ID: <HDE46LIK8GGJJ72I at other-example.com>
-MIME-Version: 1.0
-Content-Type: text/html; charset=Windows-1251
-Content-Transfer-Encoding: 8bit
-X-Spam-Flag: YES
-X-Spam-Checker-Version: SpamAssassin 2.60-cvs-jumbo.demos (1.190-2003-06-01-exp)
-X-Spam-Level: ++++++++++++++
-X-Spam-Status: Yes, hits=14.9 required=5.0 tests=BAYES_99,DATE_IN_FUTURE_06_12
-	FROM_ILLEGAL_CHARS,HTML_10_20,HTML_FONTCOLOR_UNKNOWN,HTML_FONT_BIG
-	MIME_HTML_ONLY,RCVD_IN_NJABL,SUBJ_HAS_SPACES,SUBJ_HAS_UNIQ_ID
-	SUBJ_ILLEGAL_CHARS autolearn=no version=2.60-cvs-jumbo.demos
-X-Spam-Report: 14.9 points, 5.0 required;
-	*  2.3 -- Subject contains lots of white space
-	*  1.0 -- BODY: HTML font color is unknown to us
-	*  0.3 -- BODY: FONT Size +2 and up or 3 and up
-		  [score: 1.0000]
-	*  2.8 -- BODY: Bayesian classifier spam probability is 99 to 100%
-	*  1.0 -- BODY: Message is 10% to 20% HTML
-	*  1.0 -- From contains too many raw illegal characters
-	*  1.0 -- Subject contains a unique ID
-	*  1.0 -- Subject contains too many raw illegal characters
-	*  1.2 -- Date: is 6 to 12 hours after Received: date
-		  [217.132.49.75 listed in dnsbl.njabl.org]
-	*  1.2 -- RBL: Received via a relay in dnsbl.njabl.org
-	*  2.0 -- Message only has text/html MIME parts
-Status: RO
-Content-Length: 2743
-Lines: 36
-
-<html><body><basefont face="times new roman, times, serif" size="2">
-<center>Ó÷eáíûé Öeíòp "ÊÀÄÐÛ ÄÅËÎÂÎÃÎ ÌÈÐÀ" ïpèãëaøaeò ía òpeíèíã:<br>
-<font size="5"><b>ÌÎÒÈÂÀÖÈß ÊÀÊ ÈÍÑÒÐÓÌÅÍÒ ÓÏÐÀÂËÅÍÈß ÏÅÐÑÎÍÀËÎÌ</b></font><br>
-<font color="red"><b>19 èþíÿ 2003 ã.</b></font><br>
-<b><i>Òpeíèíã ïpeäíaçía÷eí äëÿ âûcøeão è cpeäíeão óïpaâëeí÷ecêoão ïepcoíaëa.</i></b><br></center><br>
-<p align="justify"><b>Òpeíep: Áopìoòoâ Ïaâeë.</b> Ïpaêòè÷ecêèé ïcèõoëoã, oïûò paáoòû áoëee 10 ëeò â oáëacòè ïcèõoëoãèè è áèçíec-òpeíèíãoâ. Àâòop pÿäa ïóáëèêaöèé è ìeòoäè÷ecêèõ ïocoáèé paçëè÷íûõ íaïpaâëeíèé ïcèõoëoãèè, â òoì ÷ècëe: “Òeõíoëoãèÿ äeëoâoão oáùeíèÿ”, “Òeõíèêè è ïpèeìû ýôôeêòèâíûõ ïepeãoâopoâ”, “Ñòpaòeãèè ôopìèpoâaíèÿ êopïopaòèâíoão èìèäæa” è äp. Çaêoí÷èë ËÃÓ ôaêóëüòeò coöèaëüíoé ïcèõoëoãèè, Ðoccèécêóþ Àêaäeìèþ ãocóäapcòâeííoé cëóæáû ïpè Ïpeçèäeíòe ÐÔ, êópcû MBA.<br><br>
-<b><u>Öeëè òpeíèíãa:</u></b><br>
-1. Îcâoèòü ïpèeìû óïpaâëeíèÿ ìoòèâaöèeé;<br>
-2. Ïoëó÷èòü ïpaêòè÷ecêèe íaâûêè ìoòèâaöèè ïepcoíaëa ê paáoòe;<br>
-3. Îcâoèòü ocíoâíûe íaâûêè êoìaíäooápaçoâaíèÿ;<br>
-4. Îâëaäeòü ïpaêòè÷ecêèìè ìeòoäaìè coçäaíèÿ è ócèëeíèÿ paáo÷eé ìoòèâaöèè, êoìaíäooápaçoâaíèÿ.<br><br>
-<b><u>Çaäa÷è òpeíèíãa:</u></b><br>
- - Îcâoèòü ìeòoäû ïoáóæäeíèÿ äpóãèõ ëþäeé ê âûïoëíeíèþ oïpeäeëeííoé äeÿòeëüíocòè;<br>
- - Íaó÷èòücÿ íaïpaâëÿòü ïoáóæäeíèÿ coòpóäíèêoâ â cooòâeòcòâèe c çaäa÷aìè opãaíèçaöèè.<br><br>
-<b><u>Ñoäepæaíèe ïpoãpaììû:</u></b><br>
-<b>I. Ìaòepèaëüíûe è íeìaòepèaëüíûe ôopìû ìoòèâaöèè:</b><br>
-1. Ìecòo è poëü ìoòèâaöèè â óïpaâëeíèè ïepcoíaëoì;<br>
-2. Ïpaêòèêa óïpaâëeíèÿ opãaíèçaöèÿìè.<br>
-<b>II. Ïpaêòè÷ecêoe ïpèìeíeíèe ìoòèâaöèè â óïpaâëeíèè ïepcoíaëoì:</b><br>
-1. Àíòèìoòèâèpóþùèe pacïopÿæeíèÿ;<br>
-2. Ìoòèâaöèÿ è oöeíêa äeÿòeëüíocòè (poëü aòòecòaöèè coòpóäíèêoâ);<br>
-3. Ìoòèâaöèÿ è ïpaêòèêa íaêaçaíèé.<br><br>
-<b><u>Â çaâepøeíèè ïpoãpaììû ó÷acòíèêè cìoãóò:</u></b><br>
-1. Îpèeíòèpoâaòü coòpóäíèêoâ ía äocòèæeíèe oïpeäeëeííoão peçóëüòaòa;<br>
-2. Îâëaäeòü íeoáõoäèìûìè íaâûêaìè óïpaâëeíèÿ ìoòèâaöèeé ïepcoíaëa;<br>
-3. Ïpèìeíÿòü ïoëó÷eííûe çíaíèÿ â ïpaêòèêe óïpaâëeíèÿ ïepcoíaëoì;<br>
-4. Îïpeäeëÿòü èíäèâèäóaëüíûe ocoáeííocòè (ïpeäïo÷òeíèÿ) ìoòèâaöèè coòpóäíèêoâ â opãaíèçaöèè.<br>
-<i> õoäe òpeíèíãa ècïoëüçóeòcÿ paáo÷èé è cïpaâo÷íûé ìaòepèaë ïo ìoòèâaöèè è còèìóëèpoâaíèþ ïepcoíaëa poccèécêèõ êoìïaíèé. Ïo oêoí÷aíèè âûäaeòcÿ cepòèôèêaò.</i><br><br>
-<center>Ïpoäoëæèòeëüíocòü: 1 äeíü, 8 ÷acoâ (äâa ïepepûâa, oáeä)<br>
-<b>Ñòoèìocòü ó÷acòèÿ: 4 700 póáëeé áeç ÍÄÑ.</b><br>
-921-5862, 928-4156, 928-4200, 928-5321</center><br>
-<font size=1>  Åcëè èíôopìaöèÿ ïoäoáíoão poäa Âac íe èíòepecóeò è ïo äpóãèì âoïpocaì - ïèøèòe:  <a href="mailto:motiv at mailje.nl">seminar</a></font>
-<br><font size="1" color="#ffffff">3ZkRPb60QBbiHef1IRVl</font>
-</body></html>
-
-
-
diff --git a/t/mail/sendmail.t b/t/mail/sendmail.t
index 44903f3..a30517e 100644
--- a/t/mail/sendmail.t
+++ b/t/mail/sendmail.t
@@ -1,546 +1,152 @@
 use strict;
 use warnings;
-use File::Spec ();
-
-use RT::Test tests => 141;
 
-use RT::EmailParser;
-use RT::Tickets;
-use RT::Action::SendEmail;
+use RT::Test tests => undef;
 
-my @_outgoing_messages;
-my @scrips_fired;
+use File::Spec ();
+use Email::Abstract;
+use Encode;
+use utf8;
 
-#We're not testing acls here.
+# We're not testing acls here.
 my $everyone = RT::Group->new(RT->SystemUser);
 $everyone->LoadSystemInternalGroup('Everyone');
 $everyone->PrincipalObj->GrantRight( Right =>'SuperUser' );
 
-
-is (__PACKAGE__, 'main', "We're operating in the main package");
-
-{
-    no warnings qw/redefine/;
-    *RT::Action::SendEmail::SendMessage = sub {
-        my $self = shift;
-        my $MIME = shift;
-
-        main::_fired_scrip($self->ScripObj);
-        main::is(ref($MIME) , 'MIME::Entity', "hey, look. it's a mime entity");
-    };
-}
-
 # some utils
 sub first_txn    { return $_[0]->Transactions->First }
 sub first_attach { return first_txn($_[0])->Attachments->First }
-
-sub count_txns { return $_[0]->Transactions->Count }
 sub count_attachs { return first_txn($_[0])->Attachments->Count }
 
-# instrument SendEmail to pass us what it's about to send.
-# create a regular ticket
-
-my $parser = RT::EmailParser->new();
-
-# Let's test to make sure a multipart/report is processed correctly
-my $multipart_report_email = RT::Test::get_relocatable_file('multipart-report',
-    (File::Spec->updir(), 'data', 'emails'));
-my $content =  RT::Test->file_content($multipart_report_email);
-# be as much like the mail gateway as possible.
-use RT::Interface::Email;
-my %args =        (message => $content, queue => 1, action => 'correspond');
-my ($status, $msg) = RT::Interface::Email::Gateway(\%args);
-ok($status, "successfuly used Email::Gateway interface") or diag("error: $msg");
-my $tickets = RT::Tickets->new(RT->SystemUser);
-$tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-$tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
-my $tick= $tickets->First();
-isa_ok($tick, "RT::Ticket", "got a ticket object");
-ok ($tick->Id, "found ticket ".$tick->Id);
-like (first_txn($tick)->Content , qr/The original message was received/, "It's the bounce");
-
-
-# make sure it fires scrips.
-is ($#scrips_fired, 1, "Fired 2 scrips on ticket creation");
-
-undef @scrips_fired;
-
-
-
-
-$parser->ParseMIMEEntityFromScalar('From: root at localhost
-To: rt at example.com
-Subject: This is a test of new ticket creation as an unknown user
-
-Blah!
-Foob!');
-
-                                  
-use Data::Dumper;
-
-my $ticket = RT::Ticket->new(RT->SystemUser);
-my  ($id,  undef, $create_msg ) = $ticket->Create(Requestor => ['root at localhost'], Queue => 'general', Subject => 'I18NTest', MIMEObj => $parser->Entity);
-ok ($id,$create_msg);
-$tickets = RT::Tickets->new(RT->SystemUser);
-$tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-$tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
- $tick = $tickets->First();
-ok ($tick->Id, "found ticket ".$tick->Id);
-is ($tick->Subject , 'I18NTest', "failed to create the new ticket from an unprivileged account");
-
-# make sure it fires scrips.
-is ($#scrips_fired, 1, "Fired 2 scrips on ticket creation");
-# make sure it sends an autoreply
-# make sure it sends a notification to adminccs
-
-
-# we need to swap out SendMessage to test the new things we care about;
-&utf8_redef_sendmessage;
-
-# create an iso 8859-1 ticket
- at scrips_fired = ();
-
-my $iso_8859_1_ticket_email = RT::Test::get_relocatable_file(
-    'new-ticket-from-iso-8859-1', (File::Spec->updir(), 'data', 'emails'));
-$content =  RT::Test->file_content($iso_8859_1_ticket_email);
-
-
-
-$parser->ParseMIMEEntityFromScalar($content);
-
-
-# be as much like the mail gateway as possible.
-use RT::Interface::Email;
-                           
- %args =        (message => $content, queue => 1, action => 'correspond');
- RT::Interface::Email::Gateway(\%args);
- $tickets = RT::Tickets->new(RT->SystemUser);
-$tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-$tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
- $tick = $tickets->First();
-ok ($tick->Id, "found ticket ".$tick->Id);
-
-like (first_txn($tick)->Content , qr/H\x{e5}vard/, "It's signed by havard. yay");
-
-
-# make sure it fires scrips.
-is ($#scrips_fired, 1, "Fired 2 scrips on ticket creation");
-# make sure it sends an autoreply
-
-
-# make sure it sends a notification to adminccs
-
-# If we correspond, does it do the right thing to the outbound messages?
-
-$parser->ParseMIMEEntityFromScalar($content);
-  ($id, $msg) = $tick->Comment(MIMEObj => $parser->Entity);
-ok ($id, $msg);
-
-$parser->ParseMIMEEntityFromScalar($content);
-($id, $msg) = $tick->Correspond(MIMEObj => $parser->Entity);
-ok ($id, $msg);
-
-
-
-
-
-# we need to swap out SendMessage to test the new things we care about;
-&iso8859_redef_sendmessage;
-RT->Config->Set( EmailOutputEncoding => 'iso-8859-1' );
-# create an iso 8859-1 ticket
- at scrips_fired = ();
-
- $content =  RT::Test->file_content($iso_8859_1_ticket_email);
-# be as much like the mail gateway as possible.
-use RT::Interface::Email;
-                                  
- %args =        (message => $content, queue => 1, action => 'correspond');
- RT::Interface::Email::Gateway(\%args);
-$tickets = RT::Tickets->new(RT->SystemUser);
-$tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-$tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
- $tick = $tickets->First();
-ok ($tick->Id, "found ticket ".$tick->Id);
-
-like (first_txn($tick)->Content , qr/H\x{e5}vard/, "It's signed by havard. yay");
-
-
-# make sure it fires scrips.
-is ($#scrips_fired, 1, "Fired 2 scrips on ticket creation");
-# make sure it sends an autoreply
-
-
-# make sure it sends a notification to adminccs
-
-
-# If we correspond, does it do the right thing to the outbound messages?
-
-$parser->ParseMIMEEntityFromScalar($content);
- ($id, $msg) = $tick->Comment(MIMEObj => $parser->Entity);
-ok ($id, $msg);
-
-$parser->ParseMIMEEntityFromScalar($content);
-($id, $msg) = $tick->Correspond(MIMEObj => $parser->Entity);
-ok ($id, $msg);
-
-
-sub _fired_scrip {
-        my $scrip = shift;
-        push @scrips_fired, $scrip;
-}       
+sub mail_in_ticket {
+    my ($filename) = @_;
+    my $path = RT::Test::get_relocatable_file($filename,
+        (File::Spec->updir(), 'data', 'emails'));
+    my $content = RT::Test->file_content($path);
 
-sub utf8_redef_sendmessage {
-    no warnings qw/redefine/;
-    *RT::Action::SendEmail::SendMessage = sub {
-        my $self = shift;
-        my $MIME = shift;
+    RT::Test->clean_caught_mails;
+    my ($status, $id) = RT::Test->send_via_mailgate( $content );
+    ok( $status, "Fed $filename into mailgate");
 
-        my $scrip = $self->ScripObj->id;
-        ok(1, $self->ScripObj->ConditionObj->Name . " ".$self->ScripObj->ActionObj->Name);
-        main::_fired_scrip($self->ScripObj);
-        $MIME->make_singlepart;
-        main::is( ref($MIME) , 'MIME::Entity',
-                  "hey, look. it's a mime entity" );
-        main::is( ref( $MIME->head ) , 'MIME::Head',
-                  "its mime header is a mime header. yay" );
-        main::like( $MIME->head->get('Content-Type') , qr/utf-8/,
-                  "Its content type is utf-8" );
-        my $message_as_string = $MIME->bodyhandle->as_string();
-        use Encode;
-        $message_as_string = Encode::decode_utf8($message_as_string);
-        main::like(
-            $message_as_string , qr/H\x{e5}vard/,
-"The message's content contains havard's name. this will fail if it's not utf8 out");
+    my $ticket = RT::Ticket->new(RT->SystemUser);
+    $ticket->Load($id);
+    ok( $ticket->Id, "Successfully created ticket ".$ticket->Id);
 
-    };
+    my @mail = map {Email::Abstract->new($_)->cast('MIME::Entity')}
+        RT::Test->fetch_caught_mails;
+    return ($ticket, @mail);
 }
 
-sub iso8859_redef_sendmessage {
-    no warnings qw/redefine/;
-    *RT::Action::SendEmail::SendMessage = sub {
-        my $self = shift;
-        my $MIME = shift;
-
-        my $scrip = $self->ScripObj->id;
-        ok(1, $self->ScripObj->ConditionObj->Name . " ".$self->ScripObj->ActionObj->Name);
-        main::_fired_scrip($self->ScripObj);
-        $MIME->make_singlepart;
-        main::is( ref($MIME) , 'MIME::Entity',
-                  "hey, look. it's a mime entity" );
-        main::is( ref( $MIME->head ) , 'MIME::Head',
-                  "its mime header is a mime header. yay" );
-        main::like( $MIME->head->get('Content-Type') , qr/iso-8859-1/,
-                  "Its content type is iso-8859-1 - " . $MIME->head->get("Content-Type") );
-        my $message_as_string = $MIME->bodyhandle->as_string();
-        use Encode;
-        $message_as_string = Encode::decode("iso-8859-1",$message_as_string);
-        main::like(
-            $message_as_string , qr/H\x{e5}vard/, "The message's content contains havard's name. this will fail if it's not utf8 out");
-    };
+{
+    my ($ticket) = mail_in_ticket('multipart-report');
+    like( first_txn($ticket)->Content , qr/The original message was received/, "It's the bounce");
 }
 
+for my $encoding ('ISO-8859-1', 'UTF-8') {
+    RT->Config->Set( EmailOutputEncoding => $encoding );
 
- my $alt_umlaut_email = RT::Test::get_relocatable_file(
-     'multipart-alternative-with-umlaut', (File::Spec->updir(), 'data', 'emails'));
- $content =  RT::Test->file_content($alt_umlaut_email);
-
-$parser->ParseMIMEEntityFromScalar($content);
+    my ($ticket, @mail) = mail_in_ticket('new-ticket-from-iso-8859-1');
+    like (first_txn($ticket)->Content , qr/H\x{e5}vard/, "It's signed by havard. yay");
 
+    is(@mail, 1);
+    like( $mail[0]->head->get('Content-Type') , qr/$encoding/,
+          "Its content type is $encoding" );
+    my $message_as_string = $mail[0]->bodyhandle->as_string();
+    $message_as_string = Encode::decode($encoding, $message_as_string);
+    like( $message_as_string , qr/H\x{e5}vard/,
+          "The message's content contains havard's name in $encoding");
+}
 
-# be as much like the mail gateway as possible.
 {
-    no warnings qw/redefine/;
-    local *RT::Action::SendEmail::SendMessage = sub { return 1};
-
-    %args = (message => $content, queue => 1, action => 'correspond');
-    RT::Interface::Email::Gateway(\%args);
-    # TODO: following 5 lines should replaced by get_latest_ticket_ok()
-    $tickets = RT::Tickets->new(RT->SystemUser);
-    $tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-    $tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
-    $tick = $tickets->First();
-
-    ok ($tick->Id, "found ticket ".$tick->Id);
-
-    like (first_txn($tick)->Content , qr/causes Error/, "We recorded the content right as text-plain");
-    is (count_attachs($tick) , 3 , "Has three attachments, presumably a text-plain, a text-html and a multipart alternative");
-
+    my ($ticket) = mail_in_ticket('multipart-alternative-with-umlaut');
+    like( first_txn($ticket)->Content, qr/causes Error/,
+          "We recorded the content as containing 'causes error'");
+    is( count_attachs($ticket), 3,
+        "Has three attachments, presumably a text-plain, a text-html and a multipart alternative");
 }
 
-
- my $text_html_email = RT::Test::get_relocatable_file('text-html-with-umlaut',
-     (File::Spec->updir(), 'data', 'emails'));
- $content =  RT::Test->file_content($text_html_email);
-
-$parser->ParseMIMEEntityFromScalar($content);
-
-
-# be as much like the mail gateway as possible.
-&text_html_redef_sendmessage;
-
- %args =        (message => $content, queue => 1, action => 'correspond');
- RT::Interface::Email::Gateway(\%args);
- $tickets = RT::Tickets->new(RT->SystemUser);
-$tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-$tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
- $tick = $tickets->First();
-ok ($tick->Id, "found ticket ".$tick->Id);
-
-like (first_attach($tick)->Content , qr/causes Error/, "We recorded the content as containing 'causes error'") or diag( first_attach($tick)->Content );
-like (first_attach($tick)->ContentType , qr/text\/html/, "We recorded the content as text/html");
-is (count_attachs($tick), 1 , "Has one attachment, presumably a text-html and a multipart alternative");
-
-sub text_html_redef_sendmessage {
-    no warnings qw/redefine/;
-    *RT::Action::SendEmail::SendMessage = sub {
-        my $self = shift;
-        my $MIME = shift;
-        return (1) unless ($self->ScripObj->ScripActionObj->Name eq "Notify AdminCcs" );
-        is ($MIME->parts, 0, "generated correspondence mime entity
-                does not have parts");
-        is ($MIME->head->mime_type , "text/plain", "The mime type is a plain");
-    };
+{
+    my ($ticket, @mail) = mail_in_ticket('text-html-with-umlaut');
+    like( first_attach($ticket)->Content, qr/causes Error/,
+          "We recorded the content as containing 'causes error'");
+    like( first_attach($ticket)->ContentType , qr/text\/html/,
+          "We recorded the content as text/html");
+    is (count_attachs($ticket), 1,
+        "Has one attachment, just a text-html");
+
+    is(@mail, 1);
+    is( $mail[0]->parts, 0, "generated correspondence mime entity does not have parts");
+    is( $mail[0]->head->mime_type , "text/plain", "The mime type is a plain");
 }
 
-
- my $russian_email = RT::Test::get_relocatable_file('text-html-in-russian',
-     (File::Spec->updir(), 'data', 'emails'));
- $content =  RT::Test->file_content($russian_email);
-
-$parser->ParseMIMEEntityFromScalar($content);
-
-# be as much like the mail gateway as possible.
-&text_html_redef_sendmessage;
-
- %args =        (message => $content, queue => 1, action => 'correspond');
-
 {
-
-my @warnings;
-local $SIG{__WARN__} = sub {
-    push @warnings, "@_";
-};
-
-RT::Interface::Email::Gateway(\%args);
-
-TODO: {
-        local $TODO =
-'need a better approach of encoding converter, should be fixed in 4.2';
-ok( @warnings == 1 || @warnings == 2, "1 or 2 warnings are ok" );
-ok( @warnings == 1 || ( @warnings == 2 && $warnings[1] eq $warnings[0] ),
-    'if there are 2 warnings, they should be same' );
-
-like(
-    $warnings[0],
-    qr/\QEncoding error: "\x{041f}" does not map to iso-8859-1/,
-"The badly formed Russian spam we have isn't actually well-formed UTF8, which makes Encode (correctly) warn",
-);
-
-}
+    my @InputEncodings = RT->Config->Get('EmailInputEncodings');
+    RT->Config->Set( EmailInputEncodings => 'koi8-r', @InputEncodings );
+    RT->Config->Set( EmailOutputEncoding => 'koi8-r' );
+
+    my ($ticket, @mail) = mail_in_ticket('russian-subject-no-content-type');
+    like( first_attach($ticket)->ContentType, qr/text\/plain/,
+          "We recorded the content type right");
+    is( count_attachs($ticket), 1,
+        "Has one attachment, presumably a text-plain");
+    is( $ticket->Subject, "тест тест",
+        "Recorded the subject right");
+
+    is(@mail, 1);
+    is( $mail[0]->head->mime_type , "text/plain", "The only part is text/plain ");
+    like( $mail[0]->head->get("subject"), qr/\Q=?KOI8-R?B?W2V4YW1wbGUuY29tICM2XSBBdXRvUmVwbHk6INTF09Qg1MXT1A==?=\E/,
+          "The subject is encoded correctly");
+
+    RT->Config->Set(EmailInputEncodings => @InputEncodings );
+    RT->Config->Set(EmailOutputEncoding => 'utf-8');
 }
 
- $tickets = RT::Tickets->new(RT->SystemUser);
-$tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-$tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
- $tick = $tickets->First();
-ok ($tick->Id, "found ticket ".$tick->Id);
-
-like (first_attach($tick)->ContentType , qr/text\/html/, "We recorded the content right as text-html");
-
-is (count_attachs($tick) ,1 , "Has one attachment, presumably a text-html and a multipart alternative");
-
-
-
-RT->Config->Set( EmailInputEncodings => 'koi8-r', RT->Config->Get('EmailInputEncodings') );
-RT->Config->Set( EmailOutputEncoding => 'koi8-r' );
-my $russian_subject_email = RT::Test::get_relocatable_file(
-    'russian-subject-no-content-type', (File::Spec->updir(), 'data', 'emails'));
-$content = RT::Test->file_content($russian_subject_email);
-
-$parser->ParseMIMEEntityFromScalar($content);
-
-
-# be as much like the mail gateway as possible.
-&text_plain_russian_redef_sendmessage;
- %args =        (message => $content, queue => 1, action => 'correspond');
- RT::Interface::Email::Gateway(\%args);
- $tickets = RT::Tickets->new(RT->SystemUser);
-$tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-$tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
-$tick= $tickets->First();
-ok ($tick->Id, "found ticket ".$tick->Id);
-
-like (first_attach($tick)->ContentType , qr/text\/plain/, "We recorded the content type right");
-is (count_attachs($tick) ,1 , "Has one attachment, presumably a text-plain");
-is ($tick->Subject, "\x{442}\x{435}\x{441}\x{442} \x{442}\x{435}\x{441}\x{442}", "Recorded the subject right");
-sub text_plain_russian_redef_sendmessage {
-    no warnings qw/redefine/;
-    *RT::Action::SendEmail::SendMessage = sub {
-        my $self = shift; 
-        my $MIME = shift; 
-        return (1) unless ($self->ScripObj->ScripActionObj->Name eq "Notify AdminCcs" );
-        is ($MIME->head->mime_type , "text/plain", "The only part is text/plain ");
-            my $subject  = $MIME->head->get("subject");
-        chomp($subject);
-        #is( $subject ,      /^=\?KOI8-R\?B\?W2V4YW1wbGUuY39tICM3XSDUxdPUINTF09Q=\?=/ , "The $subject is encoded correctly");
-    };
+{
+    my ($ticket, @mail) = mail_in_ticket('nested-rfc-822');
+    is( $ticket->Subject, "[Jonas Liljegren] Re: [Para] Niv\x{e5}er?");
+    like( first_attach($ticket)->ContentType, qr/multipart\/mixed/,
+          "We recorded the content type right");
+    is( count_attachs($ticket), 5,
+        "Has five attachments, presumably a text-plain and a message RFC 822 and another plain");
+
+    is(@mail, 1);
+    is( $mail[0]->head->mime_type , "text/plain", "The outgoing mail is plain text");
+
+    my $encoded_subject = $mail[0]->head->get("Subject");
+    chomp $encoded_subject;
+    my $subject = decode('MIME-Header',$encoded_subject);
+    like($subject, qr/Niv\x{e5}er/, encode_utf8("The subject matches the word - $subject"));
 }
 
-my @input_encodings = RT->Config->Get( 'EmailInputEncodings' );
-shift @input_encodings;
-RT->Config->Set(EmailInputEncodings => @input_encodings );
-RT->Config->Set(EmailOutputEncoding => 'utf-8');
-
-
-
-my $nested_rfc822_email = RT::Test::get_relocatable_file('nested-rfc-822',
-    (File::Spec->updir(), 'data', 'emails'));
-$content =  RT::Test->file_content($nested_rfc822_email);
-ok ($content, "Loaded nested-rfc-822 to test");
-
-$parser->ParseMIMEEntityFromScalar($content);
-
-
-# be as much like the mail gateway as possible.
-&text_plain_nested_redef_sendmessage;
- %args =        (message => $content, queue => 1, action => 'correspond');
- RT::Interface::Email::Gateway(\%args);
- $tickets = RT::Tickets->new(RT->SystemUser);
-$tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-$tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
-$tick= $tickets->First();
-ok ($tick->Id, "found ticket ".$tick->Id);
-is ($tick->Subject, "[Jonas Liljegren] Re: [Para] Niv\x{e5}er?");
-like (first_attach($tick)->ContentType , qr/multipart\/mixed/, "We recorded the content type right");
-is (count_attachs($tick) , 5 , "Has one attachment, presumably a text-plain and a message RFC 822 and another plain");
-sub text_plain_nested_redef_sendmessage {
-    no warnings qw/redefine/;
-    *RT::Action::SendEmail::SendMessage = sub {
-        my $self = shift;
-        my $MIME = shift;
-
-        return (1) unless ($self->ScripObj->ScripActionObj->Name eq "Notify AdminCcs" );
-
-        is ($MIME->head->mime_type , "multipart/mixed", "It is a mixed multipart");
-
-        use MIME::Words qw(:all);
-        my $encoded_subject = $MIME->head->get("subject");
-        my $subject = decode_mimewords($encoded_subject);
-
-        # MIME::Words isn't actually UTF8-safe. There go 4 hours I'll never get back.
-        utf8::decode($subject);
-        like($subject, qr/Niv\x{e5}er/, "The subject matches the word - $subject");
-
-        1;
-    };
+{
+    my ($ticket) = mail_in_ticket('notes-uuencoded');
+    like( first_txn($ticket)->Content, qr/from Lotus Notes/,
+         "We recorded the content right");
+    is( count_attachs($ticket), 3, "Has three attachments");
 }
 
-
-
-
- my $uuencoded_email = RT::Test::get_relocatable_file('notes-uuencoded',
-     (File::Spec->updir(), 'data', 'emails'));
- $content =  RT::Test->file_content($uuencoded_email);
-
-$parser->ParseMIMEEntityFromScalar($content);
-
-
-# be as much like the mail gateway as possible.
 {
-    no warnings qw/redefine/;
-    local *RT::Action::SendEmail::SendMessage = sub { return 1};
-    %args =        (message => $content, queue => 1, action => 'correspond');
-    RT::Interface::Email::Gateway(\%args);
-    $tickets = RT::Tickets->new(RT->SystemUser);
-    $tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-    $tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
-    $tick= $tickets->First();
-    ok ($tick->Id, "found ticket ".$tick->Id);
-
-    like (first_txn($tick)->Content , qr/from Lotus Notes/, "We recorded the content right");
-    is (count_attachs($tick) , 3 , "Has three attachments");
+    my ($ticket) = mail_in_ticket('crashes-file-based-parser');
+    like( first_txn($ticket)->Content, qr/FYI/, "We recorded the content right");
+    is( count_attachs($ticket), 5, "Has five attachments");
 }
 
+{
+    my ($ticket) = mail_in_ticket('rt-send-cc');
+    my $cc = first_attach($ticket)->GetHeader('RT-Send-Cc');
+    like ($cc, qr/test$_/, "Found test $_") for 1..5;
+}
 
-
- my $crashes_file_based_parser_email = RT::Test::get_relocatable_file(
-     'crashes-file-based-parser', (File::Spec->updir(), 'data', 'emails'));
- $content = RT::Test->file_content($crashes_file_based_parser_email);
-
-$parser->ParseMIMEEntityFromScalar($content);
-
-
-# be as much like the mail gateway as possible.
-
-no warnings qw/redefine/;
-local *RT::Action::SendEmail::SendMessage = sub { return 1};
- %args =        (message => $content, queue => 1, action => 'correspond');
- RT::Interface::Email::Gateway(\%args);
- $tickets = RT::Tickets->new(RT->SystemUser);
-$tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-$tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
-$tick= $tickets->First();
-ok ($tick->Id, "found ticket ".$tick->Id);
-
-like (first_txn($tick)->Content , qr/FYI/, "We recorded the content right");
-is (count_attachs($tick) , 5 , "Has three attachments");
-
-
-
-
-
-
- my $rt_send_cc_email = RT::Test::get_relocatable_file('rt-send-cc',
-     (File::Spec->updir(), 'data', 'emails'));
- $content =  RT::Test->file_content($rt_send_cc_email);
-
-$parser->ParseMIMEEntityFromScalar($content);
-
-
-
- %args =        (message => $content, queue => 1, action => 'correspond');
- RT::Interface::Email::Gateway(\%args);
- $tickets = RT::Tickets->new(RT->SystemUser);
-$tickets->OrderBy(FIELD => 'id', ORDER => 'DESC');
-$tickets->Limit(FIELD => 'id' ,OPERATOR => '>', VALUE => '0');
-$tick= $tickets->First();
-ok ($tick->Id, "found ticket ".$tick->Id);
-
-my $cc = first_attach($tick)->GetHeader('RT-Send-Cc');
-like ($cc , qr/test1/, "Found test 1");
-like ($cc , qr/test2/, "Found test 2");
-like ($cc , qr/test3/, "Found test 3");
-like ($cc , qr/test4/, "Found test 4");
-like ($cc , qr/test5/, "Found test 5");
-
-
-diag q{regression test for #5248 from rt3.fsck.com};
 {
-    my $subject_folding_email = RT::Test::get_relocatable_file(
-        'subject-with-folding-ws', (File::Spec->updir(), 'data', 'emails'));
-    my $content = RT::Test->file_content($subject_folding_email);
-    my ($status, $msg, $ticket) = RT::Interface::Email::Gateway(
-        { message => $content, queue => 1, action => 'correspond' }
-    );
-    ok ($status, 'created ticket') or diag "error: $msg";
-    ok ($ticket->id, "found ticket ". $ticket->id);
+    diag "Regression test for #5248 from rt3.fsck.com";
+    my ($ticket) = mail_in_ticket('subject-with-folding-ws');
     is ($ticket->Subject, 'test', 'correct subject');
 }
 
-diag q{regression test for #5248 from rt3.fsck.com};
 {
-    my $long_subject_email = RT::Test::get_relocatable_file('very-long-subject',
-        (File::Spec->updir(), 'data', 'emails'));
-    my $content = RT::Test->file_content($long_subject_email);
-    my ($status, $msg, $ticket) = RT::Interface::Email::Gateway(
-        { message => $content, queue => 1, action => 'correspond' }
-    );
-    ok ($status, 'created ticket') or diag "error: $msg";
-    ok ($ticket->id, "found ticket ". $ticket->id);
+    diag "Regression test for #5248 from rt3.fsck.com";
+    my ($ticket) = mail_in_ticket('very-long-subject');
     is ($ticket->Subject, '0123456789'x20, 'correct subject');
 }
 
-
-
-# Don't taint the environment
-$everyone->PrincipalObj->RevokeRight(Right =>'SuperUser');
+done_testing;

commit 8bc6d5062af5ca0b0a0246ace1e67ea840bf3f10
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:07:06 2014 -0400

    Always log bytes, not characters
    
    Ensure that we always send UTF-8 encoded bytes to loggers, and not wide
    characters.  This is correctly done via an explicit call to
    Encode::encode, and not via checks of utf8::is_utf8 (which may be false
    for character strings with codepoints > 127 but < 256), and not via
    _utf8_off (which would fail similarly for such characters).

diff --git a/lib/RT.pm b/lib/RT.pm
index f10110a..e7da1a9 100644
--- a/lib/RT.pm
+++ b/lib/RT.pm
@@ -261,6 +261,9 @@ sub InitLogging {
             $frame++ while caller($frame) && caller($frame) =~ /^Log::/;
             my ($package, $filename, $line) = caller($frame);
 
+            # Encode to bytes, so we don't send wide characters
+            $p{message} = Encode::encode("UTF-8", $p{message});
+
             $p{'message'} =~ s/(?:\r*\n)+$//;
             return "[$$] [". gmtime(time) ."] [". $p{'level'} ."]: "
                 . $p{'message'} ." ($filename:$line)\n";
@@ -276,8 +279,8 @@ sub InitLogging {
             $frame++ while caller($frame) && caller($frame) =~ /^Log::/;
             my ($package, $filename, $line) = caller($frame);
 
-            # syswrite() cannot take utf8; turn it off here.
-            Encode::_utf8_off($p{message});
+            # Encode to bytes, so we don't send wide characters
+            $p{message} = Encode::encode("UTF-8", $p{message});
 
             $p{message} =~ s/(?:\r*\n)+$//;
             if ($p{level} eq 'debug') {
diff --git a/lib/RT/Action/CreateTickets.pm b/lib/RT/Action/CreateTickets.pm
index fcaeed3..a6f52ea 100644
--- a/lib/RT/Action/CreateTickets.pm
+++ b/lib/RT/Action/CreateTickets.pm
@@ -579,15 +579,11 @@ sub _ParseMultilineTemplate {
     my %args = (@_);
 
     my $template_id;
-    require Encode;
-    require utf8;
     my ( $queue, $requestor );
         $RT::Logger->debug("Line: ===");
         foreach my $line ( split( /\n/, $args{'Content'} ) ) {
             $line =~ s/\r$//;
-            $RT::Logger->debug( "Line: " . utf8::is_utf8($line)
-                ? Encode::encode_utf8($line)
-                : $line );
+            $RT::Logger->debug( "Line: $line" );
             if ( $line =~ /^===/ ) {
                 if ( $template_id && !$queue && $args{'Queue'} ) {
                     $self->{'templates'}->{$template_id}
diff --git a/share/html/Elements/Error b/share/html/Elements/Error
index b204261..d747c4e 100644
--- a/share/html/Elements/Error
+++ b/share/html/Elements/Error
@@ -78,11 +78,7 @@ $SuppressHeader => 0,
 my $error = "WebRT: $Why";
 $error .= " ($Details)" if defined $Details && length $Details;
 
-# TODO: Log::Dispatch isn't UTF-8 safe. Autrijus needs to talk to dave rolsky about getting this fixed
-use Encode ();
-Encode::_utf8_off($error);
-
-$RT::Logger->error($error);
+$RT::Logger->error( $error );
 
 if ( $session{'REST'} ) {
     $r->content_type('text/plain');

commit 5e4c0f119036d6a45179895c2e4dd803f4bd2470
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:11:33 2014 -0400

    The alluded-to deficiency is not a concern in perl ≥ 5.8.3
    
    The comment and code were added in RT 2.1.38, which only required perl
    5.6.1; the perl version was increased to 5.8.3 to cover a large number
    of encoding bugs, such as the one this comment was likely alluding to.

diff --git a/lib/RT/CurrentUser.pm b/lib/RT/CurrentUser.pm
index 9613f53..6ffe147 100644
--- a/lib/RT/CurrentUser.pm
+++ b/lib/RT/CurrentUser.pm
@@ -255,9 +255,6 @@ sub loc_fuzzy {
     my $self = shift;
     return '' if !defined $_[0] || $_[0] eq '';
 
-    # XXX: work around perl's deficiency when matching utf8 data
-    return $_[0] if Encode::is_utf8($_[0]);
-
     return $self->LanguageHandle->maketext_fuzzy( @_ );
 }
 

commit f497a1147244a1a5105604a4b1a5f748c8508df7
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:27:54 2014 -0400

    Ensure all MIME::Entity bodies are UTF-8 encoded bytes
    
    Placing wide characters into MIME::Entity objects can lead to
    double-encoding.  Always treat them as byte stores, encoding as UTF-8
    and noting their character set.
    
    In the case of Approvals/index.html, there was no need for an explicit
    MIME::Entity object; ->Correspond creates one as needed from a "Content"
    argument.

diff --git a/lib/RT/Action/CreateTickets.pm b/lib/RT/Action/CreateTickets.pm
index a6f52ea..f9525d0 100644
--- a/lib/RT/Action/CreateTickets.pm
+++ b/lib/RT/Action/CreateTickets.pm
@@ -785,10 +785,10 @@ sub ParseLines {
     );
 
     if ( $args{content} ) {
-        my $mimeobj = MIME::Entity->new();
-        $mimeobj->build(
-            Type => $args{'contenttype'} || 'text/plain',
-            Data => $args{'content'}
+        my $mimeobj = MIME::Entity->build(
+            Type    => $args{'contenttype'} || 'text/plain',
+            Charset => 'UTF-8',
+            Data    => [ map {Encode::encode( "UTF-8", $_ )} @{$args{'content'}} ],
         );
         $ticketargs{MIMEObj} = $mimeobj;
         $ticketargs{UpdateType} = $args{'updatetype'} || 'correspond';
diff --git a/lib/RT/Dashboard/Mailer.pm b/lib/RT/Dashboard/Mailer.pm
index eb620e6..5f7a99b 100644
--- a/lib/RT/Dashboard/Mailer.pm
+++ b/lib/RT/Dashboard/Mailer.pm
@@ -382,9 +382,14 @@ sub BuildEmail {
             $cid_of{$uri} = time() . $$ . int(rand(1e6));
             my ($data, $filename, $mimetype, $encoding) = GetResource($uri);
 
-            # downgrade non-text strings, because all strings are utf8 by
-            # default, which is wrong for non-text strings.
-            if ( $mimetype !~ m{text/} ) {
+            # Encode textual data in UTF-8, and downgrade (treat
+            # codepoints as codepoints, and ensure the UTF-8 flag is
+            # off) everything else.
+            my @extra;
+            if ( $mimetype =~ m{text/} ) {
+                $data = Encode::encode( "UTF-8", $data );
+                @extra = ( Charset => "UTF-8" );
+            } else {
                 utf8::downgrade( $data, 1 ) or $RT::Logger->warning("downgrade $data failed");
             }
 
@@ -396,6 +401,7 @@ sub BuildEmail {
                 Disposition  => 'inline',
                 Name         => RT::Interface::Email::EncodeToMIME( String => $filename ),
                 'Content-Id' => $cid_of{$uri},
+                @extra,
             );
 
             return "cid:$cid_of{$uri}";
diff --git a/lib/RT/Interface/Email.pm b/lib/RT/Interface/Email.pm
index ddb5be1..180e75c 100644
--- a/lib/RT/Interface/Email.pm
+++ b/lib/RT/Interface/Email.pm
@@ -268,7 +268,11 @@ sub MailError {
     my $entity = MIME::Entity->build(%entity_args);
     SetInReplyTo( Message => $entity, InReplyTo => $args{'MIMEObj'} );
 
-    $entity->attach( Data => $args{'Explanation'} . "\n" );
+    $entity->attach(
+        Type    => "text/plain",
+        Charset => "UTF-8",
+        Data    => Encode::encode( "UTF-8", $args{'Explanation'} . "\n" ),
+    );
 
     if ( $args{'MIMEObj'} ) {
         $args{'MIMEObj'}->sync_headers;
@@ -276,7 +280,7 @@ sub MailError {
     }
 
     if ( $args{'Attach'} ) {
-        $entity->attach( Data => $args{'Attach'}, Type => 'message/rfc822' );
+        $entity->attach( Data => Encode::encode( "UTF-8", $args{'Attach'} ), Type => 'message/rfc822' );
 
     }
 
@@ -760,8 +764,9 @@ sub SendForward {
                 . $txn->id ." of a ticket #". $txn->ObjectId;
         }
         $mail = MIME::Entity->build(
-            Type => 'text/plain',
-            Data => $description,
+            Type    => 'text/plain',
+            Charset => "UTF-8",
+            Data    => Encode::encode( "UTF-8", $description ),
         );
     }
 
diff --git a/lib/RT/Interface/Web.pm b/lib/RT/Interface/Web.pm
index b1f8429..f3dab23 100644
--- a/lib/RT/Interface/Web.pm
+++ b/lib/RT/Interface/Web.pm
@@ -2154,7 +2154,7 @@ sub MakeMIMEEntity {
         $Message->attach(
             Type    => $args{'Type'} || 'text/plain',
             Charset => 'UTF-8',
-            Data    => $args{'Body'},
+            Data    => Encode::encode( "UTF-8", $args{'Body'} ),
         );
     }
 
@@ -2177,7 +2177,7 @@ sub MakeMIMEEntity {
             $Message->attach(
                 Type     => $uploadinfo->{'Content-Type'},
                 Filename => $filename,
-                Data     => \@content,
+                Data     => \@content, # Bytes, as read directly from the file, above
             );
             if ( !$args{'Subject'} && !( defined $args{'Body'} && length $args{'Body'} ) ) {
                 $Message->head->set( 'Subject' => $filename );
diff --git a/lib/RT/Template.pm b/lib/RT/Template.pm
index 0507997..ec1be01 100644
--- a/lib/RT/Template.pm
+++ b/lib/RT/Template.pm
@@ -602,17 +602,17 @@ sub _DowngradeFromHTML {
 
     require HTML::FormatText;
     require HTML::TreeBuilder;
-    require Encode;
-    # need to decode_utf8, see the doc of MIMEObj method
+    # MIME objects are always bytes, not characters
     my $tree = HTML::TreeBuilder->new_from_content(
-        Encode::decode_utf8($new_entity->bodyhandle->as_string)
+        Encode::decode( 'UTF-8', $new_entity->bodyhandle->as_string)
     );
-    $new_entity->bodyhandle(MIME::Body::InCore->new(
-        \(scalar HTML::FormatText->new(
-            leftmargin  => 0,
-            rightmargin => 78,
-        )->format( $tree ))
-    ));
+    my $text = HTML::FormatText->new(
+        leftmargin  => 0,
+        rightmargin => 78,
+    )->format( $tree );
+    $text = Encode::encode( "UTF-8", $text );
+
+    $new_entity->bodyhandle(MIME::Body::InCore->new( \$text ));
     $tree->delete;
 
     $orig_entity->add_part($new_entity, 0); # plain comes before html
diff --git a/lib/RT/Test.pm b/lib/RT/Test.pm
index 19dc263..fb6cde3 100644
--- a/lib/RT/Test.pm
+++ b/lib/RT/Test.pm
@@ -826,7 +826,9 @@ sub create_ticket {
         $args{'MIMEObj'} = MIME::Entity->build(
             From    => $args{'Requestor'},
             Subject => $args{'Subject'},
-            Data    => $content,
+            Type    => "text/plain",
+            Charset => "UTF-8",
+            Data    => Encode::encode( "UTF-8", $content ),
         );
     }
 
diff --git a/lib/RT/Ticket.pm b/lib/RT/Ticket.pm
index d1fd83e..14803be 100644
--- a/lib/RT/Ticket.pm
+++ b/lib/RT/Ticket.pm
@@ -785,10 +785,10 @@ sub _Parse822HeadersForAttributes {
         }
         $args{$date} = $dateobj->ISO;
     }
-    $args{'mimeobj'} = MIME::Entity->new();
-    $args{'mimeobj'}->build(
-        Type => ( $args{'contenttype'} || 'text/plain' ),
-        Data => ($args{'content'} || '')
+    $args{'mimeobj'} = MIME::Entity->build(
+        Type    => ( $args{'contenttype'} || 'text/plain' ),
+        Charset => "UTF-8",
+        Data    => Encode::encode("UTF-8", ($args{'content'} || ''))
     );
 
     return (%args);
@@ -2257,8 +2257,11 @@ sub _RecordNote {
     }
 
     unless ( $args{'MIMEObj'} ) {
+        my $data = ref $args{'Content'}? $args{'Content'} : [ $args{'Content'} ];
         $args{'MIMEObj'} = MIME::Entity->build(
-            Data => ( ref $args{'Content'}? $args{'Content'}: [ $args{'Content'} ] )
+            Type    => "text/plain",
+            Charset => "UTF-8",
+            Data    => [ map {Encode::encode("UTF-8", $_)} @{$data} ],
         );
     }
 
@@ -2344,7 +2347,7 @@ sub DryRun {
         Type    => 'text/plain',
         Subject => defined $args{UpdateSubject} ? Encode::encode_utf8( $args{UpdateSubject} ) : "",
         Charset => 'UTF-8',
-        Data    => $args{'UpdateContent'} || "",
+        Data    => Encode::encode("UTF-8", $args{'UpdateContent'} || ""),
     );
 
     my ( $Transaction, $Description, $Object ) = $self->$action(
@@ -2373,12 +2376,12 @@ sub DryRunCreate {
     my $self = shift;
     my %args = @_;
     my $Message = MIME::Entity->build(
-        Type    => 'text/plain',
         Subject => defined $args{Subject} ? Encode::encode_utf8( $args{'Subject'} ) : "",
         (defined $args{'Cc'} ?
              ( Cc => Encode::encode_utf8( $args{'Cc'} ) ) : ()),
+        Type    => 'text/plain',
         Charset => 'UTF-8',
-        Data    => $args{'Content'} || "",
+        Data    => Encode::encode( "UTF-8", $args{'Content'} || ""),
     );
 
     my ( $Transaction, $Object, $Description ) = $self->Create(
diff --git a/share/html/Approvals/index.html b/share/html/Approvals/index.html
index 97f360a..dbdc11e 100644
--- a/share/html/Approvals/index.html
+++ b/share/html/Approvals/index.html
@@ -72,12 +72,9 @@ foreach my $arg ( keys %ARGS ) {
     next if $skip_update;
 
     if ( $ARGS{ "Approval-" . $ticket->Id . "-Notes" } ) {
-        my $notes = MIME::Entity->build(
-	    Data => [ $ARGS{ "Approval-" . $ticket->Id . "-Notes" } ]
-	);
-	RT::I18N::SetMIMEEntityToUTF8($notes); # convert text parts into utf-8
-
-        my ( $notesval, $notesmsg ) = $ticket->Correspond( MIMEObj => $notes );
+        my ( $notesval, $notesmsg ) = $ticket->Correspond(
+            Content => $ARGS{ "Approval-" . $ticket->Id . "-Notes" }
+        );
         if ($notesval) {
                 push ( @actions, loc("Approval #[_1]: Notes recorded",$ticket->Id ));
         } else {
diff --git a/share/html/REST/1.0/Forms/ticket/comment b/share/html/REST/1.0/Forms/ticket/comment
index 934cbfb..41320ba 100644
--- a/share/html/REST/1.0/Forms/ticket/comment
+++ b/share/html/REST/1.0/Forms/ticket/comment
@@ -91,8 +91,9 @@ my $ent = MIME::Entity->build(
     'X-RT-Interface' => 'REST',
 );
 $ent->attach(
-    'Content-Type' => $changes{'Content-Type'} || 'text/plain',
-    Data => $changes{Text},
+    Type    => $changes{'Content-Type'} || 'text/plain',
+    Charset => "UTF-8",
+    Data    => Encode::encode("UTF-8", $changes{Text} ),
 ) if $changes{Text};
 
 
diff --git a/share/html/REST/1.0/Forms/ticket/default b/share/html/REST/1.0/Forms/ticket/default
index 2a0c7ef..da867e7 100644
--- a/share/html/REST/1.0/Forms/ticket/default
+++ b/share/html/REST/1.0/Forms/ticket/default
@@ -196,8 +196,9 @@ else {
                     'X-RT-Interface' => 'REST',
                 );
             $v{MIMEObj}->attach(
-                Data => $text,
-                'Content-Type' => $v{'Content-Type'} || 'text/plain',
+                Type    => $v{'Content-Type'} || 'text/plain',
+                Charset => "UTF-8",
+                Data    => Encode::encode( "UTF-8", $text ),
             ) if $text;
             my ($status, $msg) = process_attachments($v{'MIMEObj'}, @atts);
             unless ($status) {
diff --git a/share/html/REST/1.0/ticket/comment b/share/html/REST/1.0/ticket/comment
index 4c058b6..177690d 100644
--- a/share/html/REST/1.0/ticket/comment
+++ b/share/html/REST/1.0/ticket/comment
@@ -108,7 +108,11 @@ my $ent = MIME::Entity->build(
     Type => "multipart/mixed",
     'X-RT-Interface' => 'REST',
 );
-$ent->attach(Data => $k->{Text}) if $k->{Text};
+$ent->attach(
+    Type    => "text/plain",
+    Charset => "UTF-8",
+    Data    => Encode::encode( "UTF-8", $k->{Text} ),
+) if $k->{Text};
 
 {
     my ($res, $msg) = process_attachments($ent, @atts);

commit 3ccd8b07673877f96da754058b70485b061170d8
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:37:42 2014 -0400

    Ensure all MIME::Entity headers are UTF-8 encoded bytes
    
    Placing wide characters into MIME::Entity objects can lead to
    double-encoding, as discovered most recently in d469cacc.  Explicitly
    decode all headers as UTF-8 when retrieving them with ->get(), and
    encode them as UTF-8 before updating them with ->set() or ->replace().
    This also applies to headers passed to ->build().  The only exceptions
    to this are fixed strings in the source (which, in the absence of "use
    utf8", are always bytes).
    
    While the majority of these headers will never have wide characters in
    them, always decoding and encoding ensures the proper disipline to
    guarantee that strings with the "UTF8" flag do not get placed in a
    header, which can cause double-encoding.

diff --git a/lib/RT/Action/SendEmail.pm b/lib/RT/Action/SendEmail.pm
index 0ad93fa..6e22817 100644
--- a/lib/RT/Action/SendEmail.pm
+++ b/lib/RT/Action/SendEmail.pm
@@ -258,7 +258,7 @@ sub Bcc {
 sub AddressesFromHeader {
     my $self      = shift;
     my $field     = shift;
-    my $header    = $self->TemplateObj->MIMEObj->head->get($field);
+    my $header    = Encode::decode("UTF-8",$self->TemplateObj->MIMEObj->head->get($field));
     my @addresses = Email::Address->parse($header);
 
     return (@addresses);
@@ -277,7 +277,7 @@ sub SendMessage {
     # ability to pass @_ to a 'post' routine.
     my ( $self, $MIMEObj ) = @_;
 
-    my $msgid = $MIMEObj->head->get('Message-ID');
+    my $msgid = Encode::decode( "UTF-8", $MIMEObj->head->get('Message-ID') );
     chomp $msgid;
 
     $self->ScripActionObj->{_Message_ID}++;
@@ -300,7 +300,7 @@ sub SendMessage {
 
     my $success = $msgid . " sent ";
     foreach (@EMAIL_RECIPIENT_HEADERS) {
-        my $recipients = $MIMEObj->head->get($_);
+        my $recipients = Encode::decode( "UTF-8", $MIMEObj->head->get($_) );
         $success .= " $_: " . $recipients if $recipients;
     }
 
@@ -531,7 +531,7 @@ sub RecordOutgoingMailTransaction {
         $type = 'EmailRecord';
     }
 
-    my $msgid = $MIMEObj->head->get('Message-ID');
+    my $msgid = Encode::decode( "UTF-8", $MIMEObj->head->get('Message-ID') );
     chomp $msgid;
 
     my ( $id, $msg ) = $transaction->Create(
@@ -649,7 +649,7 @@ sub DeferDigestRecipients {
 
         # Have to get the list of addresses directly from the MIME header
         # at this point.
-        $RT::Logger->debug( $self->TemplateObj->MIMEObj->head->as_string );
+        $RT::Logger->debug( Encode::decode( "UTF-8", $self->TemplateObj->MIMEObj->head->as_string ) );
         foreach my $rcpt ( map { $_->address } $self->AddressesFromHeader($mailfield) ) {
             next unless $rcpt;
             my $user_obj = RT::User->new(RT->SystemUser);
@@ -746,7 +746,7 @@ sub RemoveInappropriateRecipients {
     # If there are no recipients, don't try to send the message.
     # If the transaction has content and has the header RT-Squelch-Replies-To
 
-    my $msgid = $self->TemplateObj->MIMEObj->head->get('Message-Id');
+    my $msgid = Encode::decode( "UTF-8", $self->TemplateObj->MIMEObj->head->get('Message-Id') );
     if ( my $attachment = $self->TransactionObj->Attachments->First ) {
 
         if ( $attachment->GetHeader('RT-DetectedAutoGenerated') ) {
@@ -1097,12 +1097,12 @@ sub SetHeaderAsEncoding {
     my $head = $self->TemplateObj->MIMEObj->head;
 
     if ( lc($field) eq 'from' and RT->Config->Get('SMTPFrom') ) {
-        $head->replace( $field, RT->Config->Get('SMTPFrom') );
+        $head->replace( $field, Encode::encode( "UTF-8", RT->Config->Get('SMTPFrom') ) );
         return;
     }
 
-    my $value = $head->get( $field );
-    $value = $self->MIMEEncodeString( $value, $enc );
+    my $value = Encode::decode("UTF-8", $head->get( $field ));
+    $value = $self->MIMEEncodeString( $value, $enc ); # Returns bytes
     $head->replace( $field, $value );
 
 }
@@ -1112,7 +1112,8 @@ sub SetHeaderAsEncoding {
 Takes a perl string and optional encoding pass it over
 L<RT::Interface::Email/EncodeToMIME>.
 
-Basicly encode a string using B encoding according to RFC2047.
+Basicly encode a string using B encoding according to RFC2047, returning
+bytes.
 
 =cut
 
diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index 01844a7..af1f82c 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -128,19 +128,17 @@ sub Create {
     $Attachment->make_singlepart;
 
     # Get the subject
-    my $Subject = $Attachment->head->get( 'subject', 0 );
+    my $Subject = Encode::decode( 'UTF-8', $Attachment->head->get( 'subject' ) );
     $Subject = '' unless defined $Subject;
     chomp $Subject;
-    utf8::decode( $Subject ) unless utf8::is_utf8( $Subject );
 
     #Get the Message-ID
-    my $MessageId = $Attachment->head->get( 'Message-ID', 0 );
+    my $MessageId = Encode::decode( "UTF-8", $Attachment->head->get( 'Message-ID' ) );
     defined($MessageId) or $MessageId = '';
     chomp ($MessageId);
     $MessageId =~ s/^<(.*?)>$/$1/o;
 
     #Get the filename
-
     my $Filename = mime_recommended_filename($Attachment);
 
     # remove path part. 
@@ -148,8 +146,7 @@ sub Create {
 
     # MIME::Head doesn't support perl strings well and can return
     # octets which later will be double encoded in low-level code
-    my $head = $Attachment->head->as_string;
-    utf8::decode( $head ) unless utf8::is_utf8( $head );
+    my $head = Encode::decode( 'UTF-8', $Attachment->head->as_string );
 
     # If a message has no bodyhandle, that means that it has subparts (or appears to)
     # and we should act accordingly.  
diff --git a/lib/RT/Crypt/GnuPG.pm b/lib/RT/Crypt/GnuPG.pm
index d0587d4..03636c8 100644
--- a/lib/RT/Crypt/GnuPG.pm
+++ b/lib/RT/Crypt/GnuPG.pm
@@ -401,14 +401,15 @@ sub SignEncrypt {
 
     my $entity = $args{'Entity'};
     if ( $args{'Sign'} && !defined $args{'Signer'} ) {
+        my @addresses = Email::Address->parse( Encode::decode("UTF-8",$entity->head->get( 'From' )));
         $args{'Signer'} = UseKeyForSigning()
-            || (Email::Address->parse( $entity->head->get( 'From' ) ))[0]->address;
+            || $addresses[0]->address;
     }
     if ( $args{'Encrypt'} && !$args{'Recipients'} ) {
         my %seen;
         $args{'Recipients'} = [
             grep $_ && !$seen{ $_ }++, map $_->address,
-            map Email::Address->parse( $entity->head->get( $_ ) ),
+            map Email::Address->parse( Encode::decode("UTF-8",$entity->head->get( $_ ) ) ),
             qw(To Cc Bcc)
         ];
     }
@@ -520,7 +521,7 @@ sub SignEncryptRFC3156 {
         $gnupg->options->push_recipients( $_ ) foreach 
             map UseKeyForEncryption($_) || $_,
             grep !$seen{ $_ }++, map $_->address,
-            map Email::Address->parse( $entity->head->get( $_ ) ),
+            map Email::Address->parse( Encode::decode( "UTF-8", $entity->head->get( $_ ) ) ),
             qw(To Cc Bcc);
 
         my ($tmp_fh, $tmp_fn) = File::Temp::tempfile( UNLINK => 1 );
diff --git a/lib/RT/EmailParser.pm b/lib/RT/EmailParser.pm
index 0b0e5c1..fb18017 100644
--- a/lib/RT/EmailParser.pm
+++ b/lib/RT/EmailParser.pm
@@ -299,8 +299,8 @@ sub ParseCcAddressesFromHead {
 
     my (@Addresses);
 
-    my @ToObjs = Email::Address->parse( $self->Head->get('To') );
-    my @CcObjs = Email::Address->parse( $self->Head->get('Cc') );
+    my @ToObjs = Email::Address->parse( Encode::decode( "UTF-8", $self->Head->get('To') ) );
+    my @CcObjs = Email::Address->parse( Encode::decode( "UTF-8", $self->Head->get('Cc') ) );
 
     foreach my $AddrObj ( @ToObjs, @CcObjs ) {
         my $Address = $AddrObj->address;
diff --git a/lib/RT/I18N.pm b/lib/RT/I18N.pm
index bc267e4..a1dd3a3 100644
--- a/lib/RT/I18N.pm
+++ b/lib/RT/I18N.pm
@@ -231,7 +231,7 @@ sub SetMIMEEntityToEncoding {
     );
 
     # If this is a textual entity, we'd need to preserve its original encoding
-    $head->replace( "X-RT-Original-Encoding" => $charset )
+    $head->replace( "X-RT-Original-Encoding" => Encode::encode( "UTF-8", $charset ) )
 	if $head->mime_attr('content-type.charset') or IsTextualContentType($head->mime_type);
 
     return unless IsTextualContentType($head->mime_type);
@@ -243,7 +243,7 @@ sub SetMIMEEntityToEncoding {
 
         $RT::Logger->debug( "Converting '$charset' to '$enc' for "
               . $head->mime_type . " - "
-              . ( $head->get('subject') || 'Subjectless message' ) );
+              . ( Encode::decode("UTF-8",$head->get('subject')) || 'Subjectless message' ) );
 
         # NOTE:: see the comments at the end of the sub.
         Encode::_utf8_off($string);
diff --git a/lib/RT/Interface/Email.pm b/lib/RT/Interface/Email.pm
index 180e75c..64e5713 100644
--- a/lib/RT/Interface/Email.pm
+++ b/lib/RT/Interface/Email.pm
@@ -114,7 +114,7 @@ sub CheckForLoops {
     my $head = shift;
 
     # If this instance of RT sent it our, we don't want to take it in
-    my $RTLoop = $head->get("X-RT-Loop-Prevention") || "";
+    my $RTLoop = Encode::decode( "UTF-8", $head->get("X-RT-Loop-Prevention") || "" );
     chomp ($RTLoop); # remove that newline
     if ( $RTLoop eq RT->Config->Get('rtname') ) {
         return 1;
@@ -253,16 +253,17 @@ sub MailError {
     # the colons are necessary to make ->build include non-standard headers
     my %entity_args = (
         Type                    => "multipart/mixed",
-        From                    => $args{'From'},
-        Bcc                     => $args{'Bcc'},
-        To                      => $args{'To'},
-        Subject                 => $args{'Subject'},
-        'X-RT-Loop-Prevention:' => RT->Config->Get('rtname'),
+        From                    => Encode::encode( "UTF-8", $args{'From'} ),
+        Bcc                     => Encode::encode( "UTF-8", $args{'Bcc'} ),
+        To                      => Encode::encode( "UTF-8", $args{'To'} ),
+        Subject                 => EncodeToMIME( String => $args{'Subject'} ),
+        'X-RT-Loop-Prevention:' => Encode::encode( "UTF-8", RT->Config->Get('rtname') ),
     );
 
     # only set precedence if the sysadmin wants us to
     if (defined(RT->Config->Get('DefaultErrorMailPrecedence'))) {
-        $entity_args{'Precedence:'} = RT->Config->Get('DefaultErrorMailPrecedence');
+        $entity_args{'Precedence:'} =
+            Encode::encode( "UTF-8", RT->Config->Get('DefaultErrorMailPrecedence') );
     }
 
     my $entity = MIME::Entity->build(%entity_args);
@@ -378,7 +379,7 @@ sub SendEmail {
         return 0;
     }
 
-    my $msgid = $args{'Entity'}->head->get('Message-ID') || '';
+    my $msgid = Encode::decode( "UTF-8", $args{'Entity'}->head->get('Message-ID') || '' );
     chomp $msgid;
     
     # If we don't have any recipients to send to, don't send a message;
@@ -415,7 +416,7 @@ sub SendEmail {
         require RT::Date;
         my $date = RT::Date->new( RT->SystemUser );
         $date->SetToNow;
-        $args{'Entity'}->head->set( 'Date', $date->RFC2822( Timezone => 'server' ) );
+        $args{'Entity'}->head->set( 'Date', Encode::encode( "UTF-8", $date->RFC2822( Timezone => 'server' ) ) );
     }
 
     my $mail_command = RT->Config->Get('MailCommand');
@@ -518,12 +519,13 @@ sub SendEmail {
 
         # duplicate head as we want drop Bcc field
         my $head = $args{'Entity'}->head->dup;
-        my @recipients = map $_->address, map 
-            Email::Address->parse($head->get($_)), qw(To Cc Bcc);                       
+        my @recipients = map $_->address, map
+            Email::Address->parse(Encode::decode("UTF-8", $head->get($_))),
+                  qw(To Cc Bcc);
         $head->delete('Bcc');
 
         my $sender = RT->Config->Get('SMTPFrom')
-            || $args{'Entity'}->head->get('From');
+            || Encode::decode( "UTF-8", $args{'Entity'}->head->get('From') );
         chomp $sender;
 
         my $status = $smtp->mail( $sender )
@@ -631,7 +633,7 @@ sub SendEmailUsingTemplate {
     $mail->head->set( $_ => Encode::encode_utf8( $args{ $_ } ) )
         foreach grep defined $args{$_}, qw(To Cc Bcc From);
 
-    $mail->head->set( $_ => $args{ExtraHeaders}{$_} )
+    $mail->head->set( $_ => Encode::encode( "UTF-8", $args{ExtraHeaders}{$_} ) )
         foreach keys %{ $args{ExtraHeaders} };
 
     SetInReplyTo( Message => $mail, InReplyTo => $args{'InReplyTo'} );
@@ -849,7 +851,7 @@ sub SignEncrypt {
     );
     return 1 unless $args{'Sign'} || $args{'Encrypt'};
 
-    my $msgid = $args{'Entity'}->head->get('Message-ID') || '';
+    my $msgid = Encode::decode( "UTF-8", $args{'Entity'}->head->get('Message-ID') || '' );
     chomp $msgid;
 
     $RT::Logger->debug("$msgid Signing message") if $args{'Sign'};
@@ -1089,7 +1091,8 @@ sub ParseCcAddressesFromHead {
     return
         grep $_ ne $current_address && !RT::EmailParser->IsRTAddress( $_ ),
         map lc $user->CanonicalizeEmailAddress( $_->address ),
-        map RT::EmailParser->CleanupAddresses( Email::Address->parse( $args{'Head'}->get( $_ ) ) ),
+        map RT::EmailParser->CleanupAddresses( Email::Address->parse(
+              Encode::decode( "UTF-8", $args{'Head'}->get( $_ ) ) ) ),
         qw(To Cc);
 }
 
@@ -1115,7 +1118,7 @@ sub ParseSenderAddressFromHead {
 
     #Figure out who's sending this message.
     foreach my $header ( @sender_headers ) {
-        my $addr_line = $head->get($header) || next;
+        my $addr_line = Encode::decode( "UTF-8", $head->get($header) ) || next;
         my ($addr, $name) = ParseAddressFromHeader( $addr_line );
         # only return if the address is not empty
         return ($addr, $name, @errors) if $addr;
@@ -1143,7 +1146,7 @@ sub ParseErrorsToAddressFromHead {
     foreach my $header ( 'Errors-To', 'Reply-To', 'From', 'Sender' ) {
 
         # If there's a header of that name
-        my $headerobj = $head->get($header);
+        my $headerobj = Encode::decode( "UTF-8", $head->get($header) );
         if ($headerobj) {
             my ( $addr, $name ) = ParseAddressFromHeader($headerobj);
 
@@ -1188,9 +1191,9 @@ sub DeleteRecipientsFromHead {
     my %skip = map { lc $_ => 1 } @_;
 
     foreach my $field ( qw(To Cc Bcc) ) {
-        $head->set( $field =>
+        $head->set( $field => Encode::encode( "UTF-8",
             join ', ', map $_->format, grep !$skip{ lc $_->address },
-                Email::Address->parse( $head->get( $field ) )
+                Email::Address->parse( Encode::decode( "UTF-8", $head->get( $field ) ) ) )
         );
     }
 }
@@ -1223,7 +1226,7 @@ sub SetInReplyTo {
     my $get_header = sub {
         my @res;
         if ( $args{'InReplyTo'}->isa('MIME::Entity') ) {
-            @res = $args{'InReplyTo'}->head->get( shift );
+            @res = map {Encode::decode("UTF-8", $_)} $args{'InReplyTo'}->head->get( shift );
         } else {
             @res = $args{'InReplyTo'}->GetHeader( shift ) || '';
         }
@@ -1253,7 +1256,7 @@ sub SetInReplyTo {
 sub ExtractTicketId {
     my $entity = shift;
 
-    my $subject = $entity->head->get('Subject') || '';
+    my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') || '' );
     chomp $subject;
     return ParseTicketId( $subject );
 }
@@ -1458,14 +1461,14 @@ sub Gateway {
     my $head = $Message->head;
     my $ErrorsTo = ParseErrorsToAddressFromHead( $head );
     my $Sender = (ParseSenderAddressFromHead( $head ))[0];
-    my $From = $head->get("From");
+    my $From = Encode::decode( "UTF-8", $head->get("From") );
     chomp $From if defined $From;
 
-    my $MessageId = $head->get('Message-ID')
+    my $MessageId = Encode::decode( "UTF-8", $head->get('Message-ID') )
         || "<no-message-id-". time . rand(2000) .'@'. RT->Config->Get('Organization') .'>';
 
     #Pull apart the subject line
-    my $Subject = $head->get('Subject') || '';
+    my $Subject = Encode::decode( "UTF-8", $head->get('Subject') || '');
     chomp $Subject;
     
     # Lets check for mail loops of various sorts.
@@ -1488,7 +1491,7 @@ sub Gateway {
     $args{'ticket'} ||= ExtractTicketId( $Message );
 
     # ExtractTicketId may have been overridden, and edited the Subject
-    my $NewSubject = $Message->head->get('Subject');
+    my $NewSubject = Encode::decode( "UTF-8", $Message->head->get('Subject') );
     chomp $NewSubject;
 
     $SystemTicket = RT::Ticket->new( RT->SystemUser );
@@ -1736,7 +1739,7 @@ sub _RunUnsafeAction {
         @_
     );
 
-    my $From = $args{Message}->head->get("From");
+    my $From = Encode::decode( "UTF-8", $args{Message}->head->get("From") );
 
     if ( $args{'Action'} =~ /^take$/i ) {
         my ( $status, $msg ) = $args{'Ticket'}->SetOwner( $args{'CurrentUser'}->id );
@@ -1892,7 +1895,7 @@ sub _HandleMachineGeneratedMail {
         # to the scrip. We might want to notify nobody. Or just
         # the RT Owner. Or maybe all Privileged watchers.
         my ( $Sender, $junk ) = ParseSenderAddressFromHead($head);
-        $head->replace( 'RT-Squelch-Replies-To',    $Sender );
+        $head->replace( 'RT-Squelch-Replies-To',    Encode::encode("UTF-8", $Sender ) );
         $head->replace( 'RT-DetectedAutoGenerated', 'true' );
     }
     return ( 1, $ErrorsTo, "Handled machine detection", $IsALoop );
diff --git a/lib/RT/Interface/Email/Auth/GnuPG.pm b/lib/RT/Interface/Email/Auth/GnuPG.pm
index 5137707..898a8d9 100644
--- a/lib/RT/Interface/Email/Auth/GnuPG.pm
+++ b/lib/RT/Interface/Email/Auth/GnuPG.pm
@@ -118,7 +118,7 @@ sub GetCurrentUser {
     foreach my $part ( $args{'Message'}->parts_DFS ) {
         my $decrypted;
 
-        my $status = $part->head->get( 'X-RT-GnuPG-Status' );
+        my $status = Encode::decode( "UTF-8", $part->head->get( 'X-RT-GnuPG-Status' ) );
         if ( $status ) {
             for ( RT::Crypt::GnuPG::ParseStatus( $status ) ) {
                 if ( $_->{Operation} eq 'Decrypt' && $_->{Status} eq 'DONE' ) {
@@ -126,7 +126,7 @@ sub GetCurrentUser {
                 }
                 if ( $_->{Operation} eq 'Verify' && $_->{Status} eq 'DONE' ) {
                     $part->head->replace(
-                        'X-RT-Incoming-Signature' => $_->{UserString}
+                        'X-RT-Incoming-Signature' => Encode::encode( "UTF-8", $_->{UserString} )
                     );
                 }
             }
diff --git a/lib/RT/Interface/Web.pm b/lib/RT/Interface/Web.pm
index f3dab23..09c49c1 100644
--- a/lib/RT/Interface/Web.pm
+++ b/lib/RT/Interface/Web.pm
@@ -2171,16 +2171,16 @@ sub MakeMIMEEntity {
 
             my $uploadinfo = $cgi_object->uploadInfo($filehandle);
 
-            my $filename = "$filehandle";
+            my $filename = Encode::decode("UTF-8","$filehandle");
             $filename =~ s{^.*[\\/]}{};
 
             $Message->attach(
                 Type     => $uploadinfo->{'Content-Type'},
-                Filename => $filename,
+                Filename => Encode::encode("UTF-8",$filename),
                 Data     => \@content, # Bytes, as read directly from the file, above
             );
             if ( !$args{'Subject'} && !( defined $args{'Body'} && length $args{'Body'} ) ) {
-                $Message->head->set( 'Subject' => $filename );
+                $Message->head->set( 'Subject' => Encode::encode( "UTF-8", $filename ) );
             }
 
             # Attachment parts really shouldn't get a Message-ID or "interface"
diff --git a/lib/RT/Test.pm b/lib/RT/Test.pm
index fb6cde3..6ee1587 100644
--- a/lib/RT/Test.pm
+++ b/lib/RT/Test.pm
@@ -824,8 +824,8 @@ sub create_ticket {
 
     if ( my $content = delete $args{'Content'} ) {
         $args{'MIMEObj'} = MIME::Entity->build(
-            From    => $args{'Requestor'},
-            Subject => $args{'Subject'},
+            From    => Encode::encode( "UTF-8", $args{'Requestor'} ),
+            Subject => RT::Interface::Email::EncodeToMIME( String => $args{'Subject'} ),
             Type    => "text/plain",
             Charset => "UTF-8",
             Data    => Encode::encode( "UTF-8", $content ),
diff --git a/lib/RT/Ticket.pm b/lib/RT/Ticket.pm
index 14803be..751e345 100644
--- a/lib/RT/Ticket.pm
+++ b/lib/RT/Ticket.pm
@@ -2297,7 +2297,7 @@ sub _RecordNote {
     # internal Message-ID now, so all emails sent because of this
     # message have a common Message-ID
     my $org = RT->Config->Get('Organization');
-    my $msgid = $args{'MIMEObj'}->head->get('Message-ID');
+    my $msgid = Encode::decode( "UTF-8", $args{'MIMEObj'}->head->get('Message-ID') );
     unless (defined $msgid && $msgid =~ /<(rt-.*?-\d+-\d+)\.(\d+-0-0)\@\Q$org\E>/) {
         $args{'MIMEObj'}->head->set(
             'RT-Message-ID' => Encode::encode_utf8(
@@ -2309,7 +2309,7 @@ sub _RecordNote {
     #Record the correspondence (write the transaction)
     my ( $Trans, $msg, $TransObj ) = $self->_NewTransaction(
              Type => $args{'NoteType'},
-             Data => ( $args{'MIMEObj'}->head->get('subject') || 'No Subject' ),
+             Data => ( Encode::decode( "UTF-8", $args{'MIMEObj'}->head->get('Subject') ) || 'No Subject' ),
              TimeTaken => $args{'TimeTaken'},
              MIMEObj   => $args{'MIMEObj'}, 
              CommitScrips => $args{'CommitScrips'},
diff --git a/lib/RT/Util.pm b/lib/RT/Util.pm
index 9720f1d..dedade7 100644
--- a/lib/RT/Util.pm
+++ b/lib/RT/Util.pm
@@ -125,7 +125,7 @@ sub mime_recommended_filename {
     $head = $head->head if $head->isa('MIME::Entity');
 
     for my $attr_name (qw( content-disposition.filename content-type.name )) {
-        my $value = $head->mime_attr($attr_name);
+        my $value = Encode::decode("UTF-8",$head->mime_attr($attr_name));
         if ( defined $value && $value =~ /\S/ ) {
             return $value;
         }
diff --git a/sbin/rt-email-digest.in b/sbin/rt-email-digest.in
index a535e36..47cd8eb 100644
--- a/sbin/rt-email-digest.in
+++ b/sbin/rt-email-digest.in
@@ -179,8 +179,10 @@ sub send_digest {
     }
 
     # Set our sender and recipient.
-    $digest_template->MIMEObj->head->replace( 'From', RT::Config->Get('CorrespondAddress') );
-    $digest_template->MIMEObj->head->replace( 'To',   $to );
+    $digest_template->MIMEObj->head->replace(
+        'From', Encode::encode( "UTF-8", RT::Config->Get('CorrespondAddress') ) );
+    $digest_template->MIMEObj->head->replace(
+        'To',   Encode::encode( "UTF-8", $to ) );
 
     if ($print) {
         $digest_template->MIMEObj->print;
diff --git a/share/html/REST/1.0/Forms/ticket/default b/share/html/REST/1.0/Forms/ticket/default
index da867e7..33a8935 100644
--- a/share/html/REST/1.0/Forms/ticket/default
+++ b/share/html/REST/1.0/Forms/ticket/default
@@ -191,8 +191,8 @@ else {
             $v{MIMEObj} =
                 MIME::Entity->build(
                     Type => "multipart/mixed",
-                    From => $session{CurrentUser}->EmailAddress,
-                    Subject => $v{Subject},
+                    From => Encode::encode( "UTF-8", $session{CurrentUser}->EmailAddress ),
+                    Subject => Encode::encode( "UTF-8", $v{Subject}),
                     'X-RT-Interface' => 'REST',
                 );
             $v{MIMEObj}->attach(
diff --git a/share/html/Ticket/Elements/PreviewScrips b/share/html/Ticket/Elements/PreviewScrips
index 3526f31..4067c20 100644
--- a/share/html/Ticket/Elements/PreviewScrips
+++ b/share/html/Ticket/Elements/PreviewScrips
@@ -88,7 +88,7 @@ my %squelched = ProcessTransactionSquelching( \%ARGS );
               </ul>
 %         }
 %         if (RT->Config->Get('PreviewScripMessages')) {
-              <textarea cols="80" rows="5"><%$scrip->ActionObj->TemplateObj->MIMEObj->as_string%></textarea>
+              <textarea cols="80" rows="5"><% Encode::decode( "UTF-8", $scrip->ActionObj->TemplateObj->MIMEObj->as_string ) %></textarea>
 %         }
           <br />
 %     }

commit c0f4e496c606b469ab7c6c5d2d3c2c24eec0d6b9
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:39:50 2014 -0400

    Make RT::Action::SendEmail->SetHeader take characters, not bytes
    
    This helper method is used in a number of places in
    RT::Action::SendEmail, often without remembering that it should be
    passed bytes, not characters.  Change it to always take characters, and
    modify the two callsite which (correctly) passed it bytes to no longer
    do so.

diff --git a/lib/RT/Action/SendEmail.pm b/lib/RT/Action/SendEmail.pm
index 6e22817..bff6ad2 100644
--- a/lib/RT/Action/SendEmail.pm
+++ b/lib/RT/Action/SendEmail.pm
@@ -918,7 +918,8 @@ sub GetFriendlyName {
 
 =head2 SetHeader FIELD, VALUE
 
-Set the FIELD of the current MIME object into VALUE.
+Set the FIELD of the current MIME object into VALUE, which should be in
+characters, not bytes.  Returns the new header, in bytes.
 
 =cut
 
@@ -931,7 +932,7 @@ sub SetHeader {
     chomp $field;
     my $head = $self->TemplateObj->MIMEObj->head;
     $head->fold_length( $field, 10000 );
-    $head->replace( $field, $val );
+    $head->replace( $field, Encode::encode( "UTF-8", $val ) );
     return $head->get($field);
 }
 
@@ -972,7 +973,7 @@ sub SetSubject {
 
     $subject =~ s/(\r\n|\n|\s)/ /g;
 
-    $self->SetHeader( 'Subject', Encode::encode_utf8( $subject ) );
+    $self->SetHeader( 'Subject', $subject );
 
 }
 
@@ -988,11 +989,9 @@ sub SetSubjectToken {
     my $head = $self->TemplateObj->MIMEObj->head;
     $self->SetHeader(
         Subject =>
-            Encode::encode_utf8(
-                RT::Interface::Email::AddSubjectTag(
-                    Encode::decode_utf8( $head->get('Subject') ),
-                    $self->TicketObj,
-                ),
+            RT::Interface::Email::AddSubjectTag(
+                Encode::decode_utf8( $head->get('Subject') ),
+                $self->TicketObj,
             ),
     );
 }

commit 2206fe537f10607832998699e07c883e2af0da22
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 14:03:36 2014 -0400

    Add a utility method to check that an input is bytes
    
    Note that it is impossible to verify that an input is characters; here,
    we can only validate if it _could_ be bytes.
    
    First, any string with the "UTF8" flag off cannot contain codepoints
    above 255, and as such is safe.  Additionally, if the "UTF8" flag is on,
    having no codepoints above 127 means the bytes are unambigious.  Having
    codepoints above 255 is guaranteedly a sign that the input is not a byte
    string.
    
    This leaves only the case of a string with the "UTF8" flag on, and
    codepoints above 127 but below 255.  The "UTF8" flag is a sign that they
    were _likely_ touched by character data at some point.  In such cases we
    warn, suggesting that the bytes have the "UTF8" flag disabled by means
    of utf8::downgrade, if they are indeed bytes.

diff --git a/lib/RT/Util.pm b/lib/RT/Util.pm
index dedade7..f8ffccf 100644
--- a/lib/RT/Util.pm
+++ b/lib/RT/Util.pm
@@ -133,6 +133,23 @@ sub mime_recommended_filename {
     return;
 }
 
+sub assert_bytes {
+    my $string = shift;
+    return unless utf8::is_utf8($string);
+    return unless $string =~ /([^\x00-\x7F])/;
+
+    my $msg;
+    if (ord($1) > 255) {
+        $msg = "Expecting a byte string, but was passed characters";
+    } else {
+        $msg = "Expecting a byte string, but was possibly passed charcters;"
+            ." if the string is actually bytes, please use utf8::downgrade";
+    }
+    $RT::Logger->warn($msg, Carp::longmess());
+
+}
+
+
 RT::Base->_ImportOverlays();
 
 1;

commit 8e62357c4aac047cf65997dd53af54b45bc12d0d
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 14:04:45 2014 -0400

    Verify that MIME::Entity bodies are bytes, and remove _utf8_off call
    
    Use the newly-added RT::Util::assert_bytes function to verify that the
    body is indeed bytes, and not characters.
    
    We also remove the _utf8_off call -- because, contrary to what the
    comment implies, the presence or absence of the "UTF8" flag does _not_
    determine if a string is "encoded as octets and not as characters"; it
    merely states that the string is capable of holding codepoints > 255.
    If it happens to not contain any, the _utf8_off does nothing.  If it
    does, it effectively encodes all codepoints > 127 in UTF-8.
    
    Given the premise that the string contains bytes in some (probably
    non-UTF-8) encoding, re-encoding some bytes of it as UTF-8 cannot
    possibly produce valid output.  The flaw in this situation cannot be
    fixed by a simple _utf8_off, but instead must be fixed by ensuring that
    the body always contains bytes, not wide characters -- as it now does,
    thanks to the prior commits.  The call to RT::Util::assert_bytes serves
    as an additional safeguard against backsliding on that assumption.

diff --git a/lib/RT/I18N.pm b/lib/RT/I18N.pm
index a1dd3a3..9168505 100644
--- a/lib/RT/I18N.pm
+++ b/lib/RT/I18N.pm
@@ -240,13 +240,12 @@ sub SetMIMEEntityToEncoding {
 
     if ( $body && ($enc ne $charset || $enc =~ /^utf-?8(?:-strict)?$/i) ) {
         my $string = $body->as_string or return;
+        RT::Util::assert_bytes($string);
 
         $RT::Logger->debug( "Converting '$charset' to '$enc' for "
               . $head->mime_type . " - "
               . ( Encode::decode("UTF-8",$head->get('subject')) || 'Subjectless message' ) );
 
-        # NOTE:: see the comments at the end of the sub.
-        Encode::_utf8_off($string);
         Encode::from_to( $string, $charset => $enc );
 
         my $new_body = MIME::Body::InCore->new($string);
@@ -259,19 +258,6 @@ sub SetMIMEEntityToEncoding {
     }
 }
 
-# NOTES:  Why Encode::_utf8_off before Encode::from_to
-#
-# All the strings in RT are utf-8 now.  Quotes from Encode POD:
-#
-# [$length =] from_to($octets, FROM_ENC, TO_ENC [, CHECK])
-# ... The data in $octets must be encoded as octets and not as
-# characters in Perl's internal format. ...
-#
-# Not turning off the UTF-8 flag in the string will prevent the string
-# from conversion.
-
-
-
 =head2 DecodeMIMEWordsToUTF8 $raw
 
 An utility method which mimics MIME::Words::decode_mimewords, but only

commit 8140533432fbbfa097fdb4e3fd06208f1429fbee
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 14:05:12 2014 -0400

    Verify that MIME::Entity headers are bytes, and remove _utf8_off call
    
    See the prior commit for reasoning, which applies just as much to the
    header as the body.

diff --git a/lib/RT/I18N.pm b/lib/RT/I18N.pm
index 9168505..f702b3a 100644
--- a/lib/RT/I18N.pm
+++ b/lib/RT/I18N.pm
@@ -549,13 +549,13 @@ sub SetMIMEHeadToEncoding {
 
     return if $charset eq $enc and $preserve_words;
 
+    RT::Util::assert_bytes( $head->as_string );
     foreach my $tag ( $head->tags ) {
         next unless $tag; # seen in wild: headers with no name
         my @values = $head->get_all($tag);
         $head->delete($tag);
         foreach my $value (@values) {
             if ( $charset ne $enc || $enc =~ /^utf-?8(?:-strict)?$/i ) {
-                Encode::_utf8_off($value);
                 Encode::from_to( $value, $charset => $enc );
             }
             $value = DecodeMIMEWordsToEncoding( $value, $enc, $tag )

commit 2d65e31870911f036f046a3c0a7a0cd555e3e438
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:50:55 2014 -0400

    Standardize on the stricter Encode::encode("UTF-8", ...) everywhere
    
    This is not only for code consistency, but also for consistency of
    output.  Encode::encode_utf8(...) is equivalent to
    Encode::encode("utf8",...) which is the non-"strict" form of UTF-8.
    Strict UTF-8 encoding differs in that (from `perldoc Encode`):
    
        ...its range is much narrower (0 ..  0x10_FFFF to cover only 21 bits
        instead of 32 or 64 bits) and some sequences are not allowed, like
        those used in surrogate pairs, the 31 non-character code points
        0xFDD0 .. 0xFDEF, the last two code points in any plane (0xXX_FFFE
        and 0xXX_FFFF), all non-shortest encodings, etc.
    
    RT deals with interchange with databases, email, and other systems.  In
    dealing with encodings, it should ensure that it does not produce byte
    sequences that are invalid according to official Unicode standards.

diff --git a/lib/RT/Action/SendEmail.pm b/lib/RT/Action/SendEmail.pm
index bff6ad2..500ab57 100644
--- a/lib/RT/Action/SendEmail.pm
+++ b/lib/RT/Action/SendEmail.pm
@@ -990,7 +990,7 @@ sub SetSubjectToken {
     $self->SetHeader(
         Subject =>
             RT::Interface::Email::AddSubjectTag(
-                Encode::decode_utf8( $head->get('Subject') ),
+                Encode::decode( "UTF-8", $head->get('Subject') ),
                 $self->TicketObj,
             ),
     );
diff --git a/lib/RT/Dashboard/Mailer.pm b/lib/RT/Dashboard/Mailer.pm
index 5f7a99b..92f1a0d 100644
--- a/lib/RT/Dashboard/Mailer.pm
+++ b/lib/RT/Dashboard/Mailer.pm
@@ -415,16 +415,16 @@ sub BuildEmail {
     );
 
     my $entity = MIME::Entity->build(
-        From    => Encode::encode_utf8($args{From}),
-        To      => Encode::encode_utf8($args{To}),
+        From    => Encode::encode("UTF-8", $args{From}),
+        To      => Encode::encode("UTF-8", $args{To}),
         Subject => RT::Interface::Email::EncodeToMIME( String => $args{Subject} ),
         Type    => "multipart/mixed",
     );
 
     $entity->attach(
-        Data        => Encode::encode_utf8($content),
         Type        => 'text/html',
         Charset     => 'UTF-8',
+        Data        => Encode::encode("UTF-8", $content),
         Disposition => 'inline',
         Encoding    => "base64",
     );
diff --git a/lib/RT/Interface/Email.pm b/lib/RT/Interface/Email.pm
index 64e5713..bd42d3d 100644
--- a/lib/RT/Interface/Email.pm
+++ b/lib/RT/Interface/Email.pm
@@ -630,7 +630,7 @@ sub SendEmailUsingTemplate {
         return -1;
     }
 
-    $mail->head->set( $_ => Encode::encode_utf8( $args{ $_ } ) )
+    $mail->head->set( $_ => Encode::encode( "UTF-8", $args{ $_ } ) )
         foreach grep defined $args{$_}, qw(To Cc Bcc From);
 
     $mail->head->set( $_ => Encode::encode( "UTF-8", $args{ExtraHeaders}{$_} ) )
@@ -1249,8 +1249,8 @@ sub SetInReplyTo {
         if @references > 10;
 
     my $mail = $args{'Message'};
-    $mail->head->set( 'In-Reply-To' => Encode::encode_utf8(join ' ', @rtid? (@rtid) : (@id)) ) if @id || @rtid;
-    $mail->head->set( 'References' => Encode::encode_utf8(join ' ', @references) );
+    $mail->head->set( 'In-Reply-To' => Encode::encode( "UTF-8", join ' ', @rtid? (@rtid) : (@id)) ) if @id || @rtid;
+    $mail->head->set( 'References' => Encode::encode( "UTF-8", join ' ', @references) );
 }
 
 sub ExtractTicketId {
diff --git a/lib/RT/Interface/Web.pm b/lib/RT/Interface/Web.pm
index 09c49c1..a6f8a56 100644
--- a/lib/RT/Interface/Web.pm
+++ b/lib/RT/Interface/Web.pm
@@ -1483,8 +1483,12 @@ sub StoreRequestToken {
     if ($ARGS->{Attach}) {
         my $attachment = HTML::Mason::Commands::MakeMIMEEntity( AttachmentFieldName => 'Attach' );
         my $file_path = delete $ARGS->{'Attach'};
+
+        # This needs to be decoded because the value is a reference;
+        # hence it was not decoded along with all of the standard
+        # arguments in DecodeARGS
         $data->{attach} = {
-            filename => Encode::decode_utf8("$file_path"),
+            filename => Encode::decode("UTF-8", "$file_path"),
             mime     => $attachment,
         };
     }
@@ -1975,7 +1979,7 @@ sub ProcessUpdateMessage {
         Interface => RT::Interface::Web::MobileClient() ? 'Mobile' : 'Web',
     );
 
-    $Message->head->replace( 'Message-ID' => Encode::encode_utf8(
+    $Message->head->replace( 'Message-ID' => Encode::encode( "UTF-8",
         RT::Interface::Email::GenMessageId( Ticket => $args{'TicketObj'} )
     ) );
     my $old_txn = RT::Transaction->new( $session{'CurrentUser'} );
@@ -2102,7 +2106,10 @@ sub ProcessAttachments {
     {    # attachment?
         my $attachment = MakeMIMEEntity( AttachmentFieldName => 'Attach' );
 
-        my $file_path = Encode::decode_utf8("$ARGSRef->{'Attach'}");
+        # This needs to be decoded because the value is a reference;
+        # hence it was not decoded along with all of the standard
+        # arguments in DecodeARGS
+        my $file_path = Encode::decode("UTF-8", "$ARGSRef->{'Attach'}");
         $session{'Attachments'} =
           { %{ $session{'Attachments'} || {} }, $file_path => $attachment, };
     }
@@ -2140,9 +2147,9 @@ sub MakeMIMEEntity {
     );
     my $Message = MIME::Entity->build(
         Type    => 'multipart/mixed',
-        "Message-Id" => Encode::encode_utf8( RT::Interface::Email::GenMessageId ),
+        "Message-Id" => Encode::encode( "UTF-8", RT::Interface::Email::GenMessageId ),
         "X-RT-Interface" => $args{Interface},
-        map { $_ => Encode::encode_utf8( $args{ $_} ) }
+        map { $_ => Encode::encode( "UTF-8", $args{ $_} ) }
             grep defined $args{$_}, qw(Subject From Cc)
     );
 
diff --git a/lib/RT/Interface/Web/Handler.pm b/lib/RT/Interface/Web/Handler.pm
index 07e7707..0434e7c 100644
--- a/lib/RT/Interface/Web/Handler.pm
+++ b/lib/RT/Interface/Web/Handler.pm
@@ -251,7 +251,7 @@ use CGI::Emulate::PSGI;
 use Plack::Request;
 use Plack::Response;
 use Plack::Util;
-use Encode qw(encode_utf8);
+use Encode;
 
 sub PSGIApp {
     my $self = shift;
@@ -328,7 +328,7 @@ sub _psgi_response_cb {
                          $cleanup->();
                          return '';
                      }
-                     return utf8::is_utf8($_[0]) ? encode_utf8($_[0]) : $_[0];
+                     return utf8::is_utf8($_[0]) ? Encode::encode( "UTF-8", $_[0]) : $_[0];
                      return $_[0];
                  };
              });
diff --git a/lib/RT/ObjectCustomFieldValue.pm b/lib/RT/ObjectCustomFieldValue.pm
index 0e63ced..af740e9 100644
--- a/lib/RT/ObjectCustomFieldValue.pm
+++ b/lib/RT/ObjectCustomFieldValue.pm
@@ -90,7 +90,8 @@ sub Create {
     my ($val, $msg) = $cf->_CanonicalizeValue(\%args);
     return ($val, $msg) unless $val;
 
-    if ( defined $args{'Content'} && length( Encode::encode_utf8($args{'Content'}) ) > 255 ) {
+    my $encoded = Encode::encode("UTF-8", $args{'Content'});
+    if ( defined $args{'Content'} && length( $encoded ) > 255 ) {
         if ( defined $args{'LargeContent'} && length $args{'LargeContent'} ) {
             $RT::Logger->error("Content is longer than 255 bytes and LargeContent specified");
         }
diff --git a/lib/RT/Template.pm b/lib/RT/Template.pm
index ec1be01..6d92eb9 100644
--- a/lib/RT/Template.pm
+++ b/lib/RT/Template.pm
@@ -384,8 +384,8 @@ sub _Parse {
 
     ### Should we forgive normally-fatal errors?
     $parser->ignore_errors(1);
-    # MIME::Parser doesn't play well with perl strings
-    utf8::encode($content);
+    # Always provide bytes, not characters, to MIME objects
+    $content = Encode::encode( 'UTF-8', $content );
     $self->{'MIMEObj'} = eval { $parser->parse_data( \$content ) };
     if ( my $error = $@ || $parser->last_error ) {
         $RT::Logger->error( "$error" );
diff --git a/lib/RT/Ticket.pm b/lib/RT/Ticket.pm
index 751e345..b7f809e 100644
--- a/lib/RT/Ticket.pm
+++ b/lib/RT/Ticket.pm
@@ -2283,13 +2283,13 @@ sub _RecordNote {
             my $addresses = join ', ', (
                 map { RT::User->CanonicalizeEmailAddress( $_->address ) }
                     Email::Address->parse( $args{ $type . 'MessageTo' } ) );
-            $args{'MIMEObj'}->head->replace( 'RT-Send-' . $type, Encode::encode_utf8( $addresses ) );
+            $args{'MIMEObj'}->head->replace( 'RT-Send-' . $type, Encode::encode( "UTF-8", $addresses ) );
         }
     }
 
     foreach my $argument (qw(Encrypt Sign)) {
         $args{'MIMEObj'}->head->replace(
-            "X-RT-$argument" => Encode::encode_utf8( $args{ $argument } )
+            "X-RT-$argument" => Encode::encode( "UTF-8", $args{ $argument } )
         ) if defined $args{ $argument };
     }
 
@@ -2300,7 +2300,7 @@ sub _RecordNote {
     my $msgid = Encode::decode( "UTF-8", $args{'MIMEObj'}->head->get('Message-ID') );
     unless (defined $msgid && $msgid =~ /<(rt-.*?-\d+-\d+)\.(\d+-0-0)\@\Q$org\E>/) {
         $args{'MIMEObj'}->head->set(
-            'RT-Message-ID' => Encode::encode_utf8(
+            'RT-Message-ID' => Encode::encode( "UTF-8",
                 RT::Interface::Email::GenMessageId( Ticket => $self )
             )
         );
@@ -2344,8 +2344,8 @@ sub DryRun {
     }
 
     my $Message = MIME::Entity->build(
+        Subject => defined $args{UpdateSubject} ? Encode::encode( "UTF-8", $args{UpdateSubject} ) : "",
         Type    => 'text/plain',
-        Subject => defined $args{UpdateSubject} ? Encode::encode_utf8( $args{UpdateSubject} ) : "",
         Charset => 'UTF-8',
         Data    => Encode::encode("UTF-8", $args{'UpdateContent'} || ""),
     );
@@ -2376,9 +2376,9 @@ sub DryRunCreate {
     my $self = shift;
     my %args = @_;
     my $Message = MIME::Entity->build(
-        Subject => defined $args{Subject} ? Encode::encode_utf8( $args{'Subject'} ) : "",
+        Subject => defined $args{Subject} ? Encode::encode( "UTF-8", $args{'Subject'} ) : "",
         (defined $args{'Cc'} ?
-             ( Cc => Encode::encode_utf8( $args{'Cc'} ) ) : ()),
+             ( Cc => Encode::encode( "UTF-8", $args{'Cc'} ) ) : ()),
         Type    => 'text/plain',
         Charset => 'UTF-8',
         Data    => Encode::encode( "UTF-8", $args{'Content'} || ""),
diff --git a/lib/RT/Tickets.pm b/lib/RT/Tickets.pm
index 3c4ce29..de269b0 100644
--- a/lib/RT/Tickets.pm
+++ b/lib/RT/Tickets.pm
@@ -1715,7 +1715,7 @@ sub _CustomFieldLimit {
                     $self->_CloseParen;
             }
             elsif ( $op eq '=' || $op eq '!=' || $op eq '<>' ) {
-                if ( length( Encode::encode_utf8($value) ) < 256 ) {
+                if ( length( Encode::encode( "UTF-8", $value) ) < 256 ) {
                     $self->_SQLLimit(
                         ALIAS    => $ObjectCFs,
                         FIELD    => 'Content',
diff --git a/lib/RT/User.pm b/lib/RT/User.pm
index bf6f9be..50016cc 100644
--- a/lib/RT/User.pm
+++ b/lib/RT/User.pm
@@ -879,7 +879,7 @@ sub _GeneratePassword_sha512 {
 
     my $sha = Digest::SHA->new(512);
     $sha->add($salt);
-    $sha->add(encode_utf8($password));
+    $sha->add(Encode::encode( 'UTF-8', $password));
     return join("!", "", "sha512", $salt, $sha->b64digest);
 }
 
@@ -956,16 +956,16 @@ sub IsPassword {
         my $hash = MIME::Base64::decode_base64($stored);
         # Decoding yields 30 byes; first 4 are the salt, the rest are substr(SHA256,0,26)
         my $salt = substr($hash, 0, 4, "");
-        return 0 unless substr(Digest::SHA::sha256($salt . Digest::MD5::md5(encode_utf8($value))), 0, 26) eq $hash;
+        return 0 unless substr(Digest::SHA::sha256($salt . Digest::MD5::md5(Encode::encode( "UTF-8", $value))), 0, 26) eq $hash;
     } elsif (length $stored == 32) {
         # Hex nonsalted-md5
-        return 0 unless Digest::MD5::md5_hex(encode_utf8($value)) eq $stored;
+        return 0 unless Digest::MD5::md5_hex(Encode::encode( "UTF-8", $value)) eq $stored;
     } elsif (length $stored == 22) {
         # Base64 nonsalted-md5
-        return 0 unless Digest::MD5::md5_base64(encode_utf8($value)) eq $stored;
+        return 0 unless Digest::MD5::md5_base64(Encode::encode( "UTF-8", $value)) eq $stored;
     } elsif (length $stored == 13) {
         # crypt() output
-        return 0 unless crypt(encode_utf8($value), $stored) eq $stored;
+        return 0 unless crypt(Encode::encode( "UTF-8", $value), $stored) eq $stored;
     } else {
         $RT::Logger->warning("Unknown password form");
         return 0;
@@ -1054,8 +1054,7 @@ sub GenerateAuthString {
     my $self = shift;
     my $protect = shift;
 
-    my $str = $self->AuthToken . $protect;
-    utf8::encode($str);
+    my $str = Encode::encode( "UTF-8", $self->AuthToken . $protect );
 
     return substr(Digest::MD5::md5_hex($str),0,16);
 }
@@ -1072,8 +1071,7 @@ sub ValidateAuthString {
     my $auth_string = shift;
     my $protected = shift;
 
-    my $str = $self->AuthToken . $protected;
-    utf8::encode( $str );
+    my $str = Encode::encode( "UTF-8", $self->AuthToken . $protected );
 
     return $auth_string eq substr(Digest::MD5::md5_hex($str),0,16);
 }
diff --git a/share/html/NoAuth/iCal/dhandler b/share/html/NoAuth/iCal/dhandler
index 35da940..4eaa254 100644
--- a/share/html/NoAuth/iCal/dhandler
+++ b/share/html/NoAuth/iCal/dhandler
@@ -62,8 +62,8 @@ $notfound->() unless $path =~ m!^([^/]+)/([^/]+)/(.*)(\.(ical|ics))?!;
 my ($name, $auth, $search) = ($1, $2, $3);
 # Unescape parts
 $_ =~ s/\%([0-9a-z]{2})/chr(hex($1))/gei for $name, $search;
-# convert to perl strings
-$_ = Encode::decode_utf8( $_ ) for $name, $search;
+# Decode from bytes to characters
+$_ = Encode::decode( "UTF-8", $_ ) for $name, $search;
 
 my $user = RT::User->new( RT->SystemUser );
 $user->Load( $name );
diff --git a/share/html/Search/Elements/ResultsRSSView b/share/html/Search/Elements/ResultsRSSView
index d087711..41e200f 100644
--- a/share/html/Search/Elements/ResultsRSSView
+++ b/share/html/Search/Elements/ResultsRSSView
@@ -67,8 +67,8 @@ if ( $m->request_comp->path =~ RT->Config->Get('WebNoAuthRegex') ) {
     # Unescape parts
     $name =~ s/\%([0-9a-z]{2})/chr(hex($1))/gei;
 
-    # convert to perl strings
-    $name = Encode::decode_utf8($name);
+    # Decode from bytes to characters
+    $name = Encode::decode( "UTF-8", $name );
 
     my $user = RT::User->new(RT->SystemUser);
     $user->Load($name);
diff --git a/share/html/Search/Results.tsv b/share/html/Search/Results.tsv
index bcbc233..c843614 100644
--- a/share/html/Search/Results.tsv
+++ b/share/html/Search/Results.tsv
@@ -70,7 +70,7 @@ my $col_entry = sub {
     delete $col->{title}
         if $col->{title} and $col->{title} =~ /^\s*#\s*$/;
     return {
-        header => Encode::encode_utf8(loc($col->{title} || $col->{attribute})),
+        header => Encode::encode( "UTF-8", loc($col->{title} || $col->{attribute})),
         map    => $m->comp(
             "/Elements/ColumnMap",
             Name  => $col->{attribute},
@@ -127,7 +127,7 @@ while (my $row = $Tickets->Next) {
             # remove tabs from all field values, they screw up the tsv
             $val = '' unless defined $val;
             $val =~ s/(?:\n|\r)//g; $val =~ s{\t}{    }g;
-            Encode::encode_utf8($val);
+            Encode::encode( "UTF-8", $val);
         } @$col)."\n");
     }
 }
diff --git a/share/html/Ticket/Graphs/Elements/ShowGraph b/share/html/Ticket/Graphs/Elements/ShowGraph
index 1eae4b6..e9a5102 100644
--- a/share/html/Ticket/Graphs/Elements/ShowGraph
+++ b/share/html/Ticket/Graphs/Elements/ShowGraph
@@ -46,7 +46,7 @@
 %#
 %# END BPS TAGGED BLOCK }}}
 <div><img src="<% RT->Config->Get('WebPath') %>/Ticket/Graphs/<% $id %>?<% $m->comp('/Elements/QueryString', %ARGS) %>" usemap="#<% $graph->{'NAME'} || 'test' %>" style="border: none" />
-<% safe_run_child { Encode::decode_utf8( $graph->as_cmapx ) } |n %>
+<% safe_run_child { Encode::decode( "UTF-8", $graph->as_cmapx ) } |n %>
 </div>
 <& ShowLegends, %ARGS, Ticket => $ticket &>
 <%ARGS>
diff --git a/share/html/Widgets/TitleBoxStart b/share/html/Widgets/TitleBoxStart
index f6655ed..4982315 100644
--- a/share/html/Widgets/TitleBoxStart
+++ b/share/html/Widgets/TitleBoxStart
@@ -81,7 +81,7 @@ $hideable = 1 if $rolledup;
 #
 my $page = $m->request_comp->path;
 
-my $title_b64 = MIME::Base64::encode_base64(Encode::encode_utf8($title), '');
+my $title_b64 = MIME::Base64::encode_base64(Encode::encode( "UTF-8", $title), '');
 
 my $tid  = "TitleBox--$page--" .
             join '--', ($class, $bodyclass, $title_b64, $id);
diff --git a/t/00-mason-syntax.t b/t/00-mason-syntax.t
index 0f77876..e4ea007 100644
--- a/t/00-mason-syntax.t
+++ b/t/00-mason-syntax.t
@@ -20,12 +20,12 @@ use HTML::Mason;
 use HTML::Mason::Compiler;
 use HTML::Mason::Compiler::ToObject;
 BEGIN { require RT::Test; }
-use Encode qw(decode_utf8);
+use Encode;
 
 sub compile_file {
     my $file = shift;
 
-    my $text = decode_utf8(RT::Test->file_content($file));
+    my $text = Encode::decode( "UTF-8", RT::Test->file_content($file));
 
     my $compiler = new HTML::Mason::Compiler::ToObject;
     $compiler->compile(
diff --git a/t/api/attachment.t b/t/api/attachment.t
index 8b7cb60..4d607e6 100644
--- a/t/api/attachment.t
+++ b/t/api/attachment.t
@@ -61,7 +61,7 @@ is ($#headers, 2, "testing a bunch of singline multiple headers" );
     require Encode;
     is(
         Encode::decode( 'iso-8859-1', $mime->stringify_body ),
-        Encode::decode( 'utf8',       "HÃ¥vard\n" ),
+        Encode::decode( 'UTF-8',      "HÃ¥vard\n" ),
         'body of ContentAsMIME is original'
     );
 }
diff --git a/t/api/canonical_charset.t b/t/api/canonical_charset.t
index a426d89..8658723 100644
--- a/t/api/canonical_charset.t
+++ b/t/api/canonical_charset.t
@@ -22,7 +22,7 @@ for my $charset ( keys %map ) {
 
 my $mime   = MIME::Entity->build(
     Type => 'text/plain; charset=gb2312',
-    Data => [encode('gbk', decode_utf8("法新社倫敦11日電"))],
+    Data => [Encode::encode("gbk", Encode::decode( "UTF-8", "法新社倫敦11日電"))],
 );
 
 RT::I18N::SetMIMEEntityToUTF8($mime);
diff --git a/t/api/password-types.t b/t/api/password-types.t
index 10a874a..292f2b4 100644
--- a/t/api/password-types.t
+++ b/t/api/password-types.t
@@ -43,7 +43,7 @@ like($root->__Value("Password"), qr/^\!$default\!/, "And is now upgraded to salt
 
 # Non-ASCII salted truncated SHA-256
 my $non_ascii_trunc = MIME::Base64::encode_base64(
-    "salt" . substr(Digest::SHA::sha256("salt".Digest::MD5::md5(encode_utf8("áěšý"))),0,26),
+    "salt" . substr(Digest::SHA::sha256("salt".Digest::MD5::md5(Encode::encode("UTF-8","áěšý"))),0,26),
     ""
 );
 $root->_Set( Field => "Password", Value => $non_ascii_trunc);
diff --git a/t/mail/charsets-outgoing.t b/t/mail/charsets-outgoing.t
index 2fc91f2..165f823 100644
--- a/t/mail/charsets-outgoing.t
+++ b/t/mail/charsets-outgoing.t
@@ -72,7 +72,7 @@ foreach my $set ( 'ru', 'latin1' ) {
     my $status = 1;
     foreach my $mail ( @mails ) {
         my $entity = parse_mail( $mail );
-        my $subject = Encode::decode_utf8( $entity->head->get('Subject') );
+        my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
         $subject =~ /$string{$set}{test}/
             or do { $status = 0; diag "wrong subject: $subject" };
     }
@@ -101,7 +101,7 @@ diag "ascii subject with non-ascii subject tag";
     my $status = 1;
     foreach my $mail ( @mails ) {
         my $entity = parse_mail( $mail );
-        my $subject = Encode::decode_utf8( $entity->head->get('Subject') );
+        my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
         $subject =~ /$string{$tag_set}{support}/
             or do { $status = 0; diag "wrong subject: $subject" };
     }
@@ -122,7 +122,7 @@ foreach my $set ( 'ru', 'latin1' ) {
     my $status = 1;
     foreach my $mail ( @mails ) {
         my $entity = parse_mail( $mail );
-        my $subject = Encode::decode_utf8( $entity->head->get('Subject') );
+        my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
         $subject =~ /$string{$tag_set}{support}/
             or do { $status = 0; diag "wrong subject: $subject" };
         $subject =~ /$string{$set}{test}/
@@ -171,7 +171,7 @@ diag "ascii subject with non-ascii subject prefix in template";
     my $status = 1;
     foreach my $mail ( @mails ) {
         my $entity = parse_mail( $mail );
-        my $subject = Encode::decode_utf8( $entity->head->get('Subject') );
+        my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
         $subject =~ /$string{$prefix_set}{autoreply}/
             or do { $status = 0; diag "wrong subject: $subject" };
     }
@@ -192,7 +192,7 @@ foreach my $set ( 'ru', 'latin1' ) {
     my $status = 1;
     foreach my $mail ( @mails ) {
         my $entity = parse_mail( $mail );
-        my $subject = Encode::decode_utf8( $entity->head->get('Subject') );
+        my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
         $subject =~ /$string{$prefix_set}{autoreply}/
             or do { $status = 0; diag "wrong subject: $subject" };
         $subject =~ /$string{$set}{test}/
@@ -222,7 +222,7 @@ foreach my $set ( 'ru', 'latin1' ) {
     my $status = 1;
     foreach my $mail ( @mails ) {
         my $entity = parse_mail( $mail );
-        my $subject = Encode::decode_utf8( $entity->head->get('Subject') );
+        my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
         $subject =~ /$string{$prefix_set}{autoreply}/
             or do { $status = 0; diag "wrong subject: $subject" };
         $subject =~ /$string{$tag_set}{support}/
@@ -275,7 +275,7 @@ foreach my $set ( 'ru', 'latin1' ) {
     my $status = 1;
     foreach my $mail ( @mails ) {
         my $entity = parse_mail( $mail );
-        my $subject = Encode::decode_utf8( $entity->head->get('Subject') );
+        my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
         $subject =~ /$string{$set}{test}/
             or do { $status = 0; diag "wrong subject: $subject" };
     }
@@ -303,7 +303,7 @@ foreach my $set ( 'ru', 'latin1' ) {
     my $status = 1;
     foreach my $mail ( @mails ) {
         my $entity = parse_mail( $mail );
-        my $subject = Encode::decode_utf8( $entity->head->get('Subject') );
+        my $subject = Encode::decode( "UTF-8", $entity->head->get('Subject') );
         $subject =~ /$string{$set}{test}/
             or do { $status = 0; diag "wrong subject: $subject" };
         $subject =~ /$string{$tag_set}{support}/
diff --git a/t/mail/gateway.t b/t/mail/gateway.t
index 9482ffc..4f906c8 100644
--- a/t/mail/gateway.t
+++ b/t/mail/gateway.t
@@ -504,8 +504,7 @@ EOF
     is ($tick->Id, $id, "correct ticket");
     is ($tick->Subject , 'This is a test of I18N ticket creation', "Created the ticket - ". $tick->Subject);
 
-    my $unistring = "\303\241\303\251\303\255\303\263\303\272";
-    Encode::_utf8_on($unistring);
+    my $unistring = Encode::decode("UTF-8","\303\241\303\251\303\255\303\263\303\272");
     is (
         $tick->Transactions->First->Content,
         $tick->Transactions->First->Attachments->First->Content,
@@ -542,8 +541,7 @@ EOF
     is ($tick->Id, $id, "correct ticket");
     is ($tick->Subject , 'This is a test of I18N ticket creation', "Created the ticket");
 
-    my $unistring = "\303\241\303\251\303\255\303\263\303\272";
-    Encode::_utf8_on($unistring);
+    my $unistring = Encode::decode("UTF-8","\303\241\303\251\303\255\303\263\303\272");
 
     ok (
         $tick->Transactions->First->Content =~ $unistring,
@@ -573,8 +571,7 @@ EOF
     my $tick = RT::Test->last_ticket;
     is ($tick->Id, $id, "correct ticket");
 
-    my $content = $tick->Transactions->First->Content;
-    Encode::_utf8_off($content);
+    my $content = Encode::encode("UTF-8",$tick->Transactions->First->Content);
 
     like $content, qr{informaci\303\263n confidencial};
     like $content, qr{informaci\357\277\275n confidencial};
diff --git a/t/mail/header-characters.t b/t/mail/header-characters.t
index fe06d1a..36edfc3 100644
--- a/t/mail/header-characters.t
+++ b/t/mail/header-characters.t
@@ -14,7 +14,7 @@ SKIP:{
       . "you have $Email::Address::VERSION", 3,
       if $Email::Address::VERSION < 1.893;
 
-    my $mail = encode( 'iso-8859-1', <<'.' );
+    my $mail = Encode::encode( 'iso-8859-1', <<'.' );
 From: René@example.com>
 Reply-To: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
 Subject: testing non-ASCII From
@@ -42,7 +42,7 @@ SKIP:{
       . "you have $Email::Address::VERSION", 3,
       if $Email::Address::VERSION < 1.893;
 
-    my $mail = encode( 'iso-8859-1', <<'.' );
+    my $mail = Encode::encode( 'iso-8859-1', <<'.' );
 From: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
 Reply-To: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
 Subject: testing non-ASCII From
diff --git a/t/mail/sendmail.t b/t/mail/sendmail.t
index a30517e..3d4164f 100644
--- a/t/mail/sendmail.t
+++ b/t/mail/sendmail.t
@@ -115,7 +115,7 @@ for my $encoding ('ISO-8859-1', 'UTF-8') {
     my $encoded_subject = $mail[0]->head->get("Subject");
     chomp $encoded_subject;
     my $subject = decode('MIME-Header',$encoded_subject);
-    like($subject, qr/Niv\x{e5}er/, encode_utf8("The subject matches the word - $subject"));
+    like($subject, qr/Niv\x{e5}er/, Encode::encode("UTF-8", "The subject matches the word - $subject"));
 }
 
 {
diff --git a/t/web/attachment_encoding.t b/t/web/attachment_encoding.t
index 5af7fda..007e5bd 100644
--- a/t/web/attachment_encoding.t
+++ b/t/web/attachment_encoding.t
@@ -44,7 +44,7 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
 {
 
     my $file =
-      File::Spec->catfile( RT::Test->temp_directory, encode_utf8 '附件.txt' );
+      File::Spec->catfile( RT::Test->temp_directory, Encode::encode("UTF-8",'附件.txt') );
     open( my $fh, '>', $file ) or die $!;
     binmode $fh, ':utf8';
     print $fh '附件';
@@ -59,7 +59,7 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
     );
     $m->content_like( qr/Ticket \d+ created/i, 'created the ticket' );
     $m->content_contains( '附件.txt', 'attached filename' );
-    $m->content_lacks( encode_utf8 '附件.txt', 'no double encoded attached filename' );
+    $m->content_lacks( Encode::encode("UTF-8",'附件.txt'), 'no double encoded attached filename' );
     $m->follow_link_ok( { text => 'with headers' },
         '-> /Ticket/Attachment/WithHeaders/...' );
 
diff --git a/t/web/basic.t b/t/web/basic.t
index e61e80e..430d8c8 100644
--- a/t/web/basic.t
+++ b/t/web/basic.t
@@ -27,7 +27,7 @@ my $url = $agent->rt_base_url;
     $agent->goto_create_ticket(1);
     is ($agent->status, 200, "Loaded Create.html");
     $agent->form_name('TicketCreate');
-    my $string = Encode::decode_utf8("I18N Web Testing æøå");
+    my $string = Encode::decode("UTF-8","I18N Web Testing æøå");
     $agent->field('Subject' => "Ticket with utf8 body");
     $agent->field('Content' => $string);
     ok($agent->submit, "Created new ticket with $string as Content");
@@ -49,7 +49,7 @@ my $url = $agent->rt_base_url;
     is ($agent->status, 200, "Loaded Create.html");
     $agent->form_name('TicketCreate');
 
-    my $string = Encode::decode_utf8("I18N Web Testing æøå");
+    my $string = Encode::decode( "UTF-8","I18N Web Testing æøå");
     $agent->field('Subject' => $string);
     $agent->field('Content' => "Ticket with utf8 subject");
     ok($agent->submit, "Created new ticket with $string as Content");
diff --git a/t/web/html_template.t b/t/web/html_template.t
index 78b95a3..3dbda63 100644
--- a/t/web/html_template.t
+++ b/t/web/html_template.t
@@ -61,7 +61,7 @@ diag('test real mail outgoing') if $ENV{TEST_VERBOSE};
 
     # $mail is utf8 encoded
     my ($mail) = RT::Test->fetch_caught_mails;
-    $mail = decode_utf8 $mail;
+    $mail = Encode::decode("UTF-8", $mail );
     like( $mail, qr/你好.*你好/s,    'mail has éèà€' );
     like( $mail, qr/éèà€.*éèà€/s, 'mail has éèà€' );
     like( $mail, qr/标题.*标题/s,    'mail has ticket subject 标题' );
diff --git a/t/web/offline_messages_utf8.t b/t/web/offline_messages_utf8.t
index 4518c7b..8a54140 100644
--- a/t/web/offline_messages_utf8.t
+++ b/t/web/offline_messages_utf8.t
@@ -35,7 +35,7 @@ EOF
         fields    => { string => $template, },
         button    => 'UpdateTickets',
     );
-    my $content = encode 'utf8', $m->content;
+    my $content = Encode::encode("UTF-8", $m->content);
     ok( $content =~ m/申請單 #(\d+) 成功新增於 'General' 表單/, 'message is shown right' );
     $ticket_id = $1;
 }
@@ -55,7 +55,7 @@ EOF
         button    => 'UpdateTickets',
     );
 
-    my $content = encode 'utf8', $m->content;
+    my $content = Encode::encode("UTF-8", $m->content);
     ok(
         $content =~
 qr/主題\s*的值從\s*'test message'\s*改為\s*'test message update'/,
diff --git a/t/web/rest-non-ascii-subject.t b/t/web/rest-non-ascii-subject.t
index 8b870a8..bc0b53e 100644
--- a/t/web/rest-non-ascii-subject.t
+++ b/t/web/rest-non-ascii-subject.t
@@ -32,8 +32,7 @@ Text: $text";
 $m->post("$baseurl/REST/1.0/ticket/new", [
     user    => 'root',
     pass    => 'password',
-# error message from HTTP::Message: content must be bytes
-    content => Encode::encode_utf8($content),
+    content => Encode::encode( "UTF-8", $content),
 ], Content_Type => 'form-data' );
 
 my ($id) = $m->content =~ /Ticket (\d+) created/;

commit ecf4e7c1b6ab429004d54286e5f917c058957dcb
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:51:54 2014 -0400

    Remove "use utf8" from RT::I18N::fr, making NBSP explicit
    
    "use utf8" causes the sourcecode (including all strings) to be
    interpreted by perl as characters encoded in UTF-8, not bytes.  In
    lib/RT/I18N/fr.pm, this was being used to substitude codepoint 160
    (NO-BREAK SPACE, U+00A0) for commas.  The fact that the space character
    was not 0x20, but rather 0xA0, was mostly hidden by use of "use utf8".
    
    Remove the "use utf8" and make the replacement character clear.

diff --git a/lib/RT/I18N/fr.pm b/lib/RT/I18N/fr.pm
index 23b047a..904b841 100644
--- a/lib/RT/I18N/fr.pm
+++ b/lib/RT/I18N/fr.pm
@@ -48,7 +48,6 @@
 
 use strict;
 use warnings;
-use utf8;
 
 package RT::I18N::fr;
 use base 'RT::I18N';
@@ -59,8 +58,8 @@ use warnings;
 sub numf {
 	my ($handle, $num) = @_[0,1];
 	my $fr_num = $handle->SUPER::numf($num);
-	# French prefer to print 1000 as 1 000 rather than 1,000
-	$fr_num =~ tr<.,><, >;
+	# French prefer to print 1000 as 1(nbsp)000 rather than 1,000
+	$fr_num =~ tr<.,><,\x{A0}>;
 	return $fr_num;
 }
 

commit b3c6ae69a5813948c9e884369e20a4003b1f15fc
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:54:44 2014 -0400

    Remove remaining cases of "use utf8"
    
    All remaining cases of "use utf8" lie in the testsuite.  As "use utf8"
    changes the semantics of dealing with Unicode strings, remove it to
    allow programmers to always assume that literals are interpreted as
    bytestrings, not characters.  To do otherwise means that one must always
    ask if "use utf8" is in scope before performing operations on any
    literals; instead, simply make the encodings and decodings explicit.
    
    Note that wide characters may appear in editors, and that the encoding
    of the characters on _disk_ will always be UTF-8.  The removal of "use
    utf8" merely means that perl will generate a two-byte string from "é",
    and not a one-character string.

diff --git a/t/api/password-types.t b/t/api/password-types.t
index 292f2b4..22b6c91 100644
--- a/t/api/password-types.t
+++ b/t/api/password-types.t
@@ -3,8 +3,7 @@ use warnings;
 
 use RT::Test;
 use Digest::MD5;
-use Encode 'encode_utf8';
-use utf8;
+use Encode;
 
 my $default = "sha512";
 
@@ -43,9 +42,9 @@ like($root->__Value("Password"), qr/^\!$default\!/, "And is now upgraded to salt
 
 # Non-ASCII salted truncated SHA-256
 my $non_ascii_trunc = MIME::Base64::encode_base64(
-    "salt" . substr(Digest::SHA::sha256("salt".Digest::MD5::md5(Encode::encode("UTF-8","áěšý"))),0,26),
+    "salt" . substr(Digest::SHA::sha256("salt".Digest::MD5::md5("áěšý")),0,26),
     ""
 );
 $root->_Set( Field => "Password", Value => $non_ascii_trunc);
-ok($root->IsPassword("áěšý"), "Unsalted MD5 base64 works");
+ok($root->IsPassword(Encode::decode("UTF-8", "áěšý")), "Unsalted MD5 base64 works");
 like($root->__Value("Password"), qr/^\!$default\!/, "And is now upgraded to salted $default");
diff --git a/t/i18n/default.t b/t/i18n/default.t
index ea0848f..8d8f98b 100644
--- a/t/i18n/default.t
+++ b/t/i18n/default.t
@@ -13,10 +13,11 @@ $m->content_contains('<html lang="en">');
 
 $m->add_header('Accept-Language' => 'zh-tw,zh;q=0.8,en-gb;q=0.5,en;q=0.3');
 $m->get_ok('/');
-use utf8;
 Encode::_utf8_on($m->{content});
-$m->title_is('登入', 'Page title properly translated to chinese');
-$m->content_contains('密碼','Password properly translated');
+$m->title_is( Encode::decode("UTF-8",'登入'),
+              'Page title properly translated to chinese');
+$m->content_contains( Encode::decode("UTF-8",'密碼'),
+                      'Password properly translated');
 {
     local $TODO = "We fail to correctly advertise the langauage in the <html> block";
     $m->content_contains('<html lang="zh-tw">');
diff --git a/t/mail/dashboard-chart-with-utf8.t b/t/mail/dashboard-chart-with-utf8.t
index 79f5f0e..37f8ce0 100644
--- a/t/mail/dashboard-chart-with-utf8.t
+++ b/t/mail/dashboard-chart-with-utf8.t
@@ -12,8 +12,6 @@ BEGIN {
     }
 }
 
-use utf8;
-
 my $root = RT::Test->load_or_create_user( Name => 'root' );
 
 my ( $baseurl, $m ) = RT::Test->started_ok;
@@ -21,11 +19,11 @@ ok( $m->login, 'logged in' );
 my $ticket = RT::Ticket->new( $RT::SystemUser );
 $ticket->Create(
     Queue   => 'General',
-    Subject => 'test äöü',
+    Subject => Encode::decode("UTF-8",'test äöü'),
 );
 ok( $ticket->id, 'created ticket' );
 
-$m->get_ok(q{/Search/Chart.html?Query=Subject LIKE 'test äöü'});
+$m->get_ok(Encode::decode("UTF-8", q{/Search/Chart.html?Query=Subject LIKE 'test äöü'}));
 $m->submit_form(
     form_name => 'SaveSearch',
     fields    => {
@@ -58,7 +56,7 @@ $m->field( 'Hour'      => '06:00' );
 $m->click_button( name => 'Save' );
 $m->content_contains('Subscribed to dashboard dashboard foo');
 
-my $c     = $m->get(q{/Search/Chart?Query=Subject LIKE 'test äöü'});
+my $c     = $m->get(Encode::decode("UTF-8",q{/Search/Chart?Query=Subject LIKE 'test äöü'}));
 my $image = $c->content;
 RT::Test->run_and_capture(
     command => $RT::SbinPath . '/rt-email-dashboards', all => 1
diff --git a/t/mail/extractsubjecttag.t b/t/mail/extractsubjecttag.t
index 14fab44..1aadaa7 100644
--- a/t/mail/extractsubjecttag.t
+++ b/t/mail/extractsubjecttag.t
@@ -1,6 +1,5 @@
 use strict;
 use warnings;
-use utf8;
 
 use RT::Test tests => 18;
 
diff --git a/t/mail/header-characters.t b/t/mail/header-characters.t
index 36edfc3..63ad525 100644
--- a/t/mail/header-characters.t
+++ b/t/mail/header-characters.t
@@ -3,7 +3,6 @@ use warnings;
 
 use RT::Test tests => 12;
 use Test::Warn;
-use utf8;
 use Encode;
 
 my ($baseurl, $m) = RT::Test->started_ok;
@@ -14,7 +13,7 @@ SKIP:{
       . "you have $Email::Address::VERSION", 3,
       if $Email::Address::VERSION < 1.893;
 
-    my $mail = Encode::encode( 'iso-8859-1', <<'.' );
+    my $mail = Encode::encode( 'iso-8859-1', Encode::decode( "UTF-8", <<'.') );
 From: René@example.com>
 Reply-To: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
 Subject: testing non-ASCII From
@@ -42,7 +41,7 @@ SKIP:{
       . "you have $Email::Address::VERSION", 3,
       if $Email::Address::VERSION < 1.893;
 
-    my $mail = Encode::encode( 'iso-8859-1', <<'.' );
+    my $mail = Encode::encode( 'iso-8859-1', Encode::decode( "UTF-8", <<'.' ) );
 From: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
 Reply-To: =?iso-8859-1?Q?Ren=E9?= <René@example.com>
 Subject: testing non-ASCII From
diff --git a/t/mail/one-time-recipients.t b/t/mail/one-time-recipients.t
index 3484d14..a9881cd 100644
--- a/t/mail/one-time-recipients.t
+++ b/t/mail/one-time-recipients.t
@@ -1,6 +1,5 @@
 use strict;
 use warnings;
-use utf8;
 
 use RT::Test tests => 38;
 
diff --git a/t/mail/rfc2231-attachment.t b/t/mail/rfc2231-attachment.t
index fc74c47..9610961 100644
--- a/t/mail/rfc2231-attachment.t
+++ b/t/mail/rfc2231-attachment.t
@@ -1,7 +1,6 @@
 use strict;
 use warnings;
 
-use utf8;
 use RT::Test tests => undef;
 my ($baseurl, $m) = RT::Test->started_ok;
 ok $m->login, 'logged in as root';
@@ -20,7 +19,7 @@ diag "encoded attachment filename with parameter continuations";
     ok( $id, "Created ticket" );
 
     $m->get_ok("/Ticket/Display.html?id=$id");
-    $m->content_contains("新しいテキスト ドキュメント.txt", "found full filename");
+    $m->content_contains(Encode::decode("UTF-8","新しいテキスト ドキュメント.txt"), "found full filename");
 }
 
 undef $m;
diff --git a/t/mail/sendmail.t b/t/mail/sendmail.t
index 3d4164f..acfedce 100644
--- a/t/mail/sendmail.t
+++ b/t/mail/sendmail.t
@@ -6,7 +6,6 @@ use RT::Test tests => undef;
 use File::Spec ();
 use Email::Abstract;
 use Encode;
-use utf8;
 
 # We're not testing acls here.
 my $everyone = RT::Group->new(RT->SystemUser);
@@ -89,7 +88,7 @@ for my $encoding ('ISO-8859-1', 'UTF-8') {
           "We recorded the content type right");
     is( count_attachs($ticket), 1,
         "Has one attachment, presumably a text-plain");
-    is( $ticket->Subject, "тест тест",
+    is( $ticket->Subject, Encode::decode("UTF-8","тест тест"),
         "Recorded the subject right");
 
     is(@mail, 1);
diff --git a/t/mail/threading.t b/t/mail/threading.t
index 773b720..9d3a077 100644
--- a/t/mail/threading.t
+++ b/t/mail/threading.t
@@ -1,6 +1,5 @@
 use strict;
 use warnings;
-use utf8;
 
 use RT::Test tests => 22;
 RT->Config->Set( NotifyActor => 1 );
diff --git a/t/mail/wrong_mime_charset.t b/t/mail/wrong_mime_charset.t
index 530b5f3..7880149 100644
--- a/t/mail/wrong_mime_charset.t
+++ b/t/mail/wrong_mime_charset.t
@@ -3,9 +3,8 @@ use warnings;
 use RT::Test nodb => 1, tests => 6;
 
 use_ok('RT::I18N');
-use utf8;
 use Encode;
-my $test_string    = 'À';
+my $test_string    = Encode::decode("UTF-8", 'À');
 my $encoded_string = encode( 'iso-8859-1', $test_string );
 my $mime           = MIME::Entity->build(
     "Subject" => $encoded_string,
diff --git a/t/web/attachment_encoding.t b/t/web/attachment_encoding.t
index 007e5bd..fb768dc 100644
--- a/t/web/attachment_encoding.t
+++ b/t/web/attachment_encoding.t
@@ -7,10 +7,12 @@ use Encode;
 my ( $baseurl, $m ) = RT::Test->started_ok;
 ok $m->login, 'logged in as root';
 
-use utf8;
-
 use File::Spec;
 
+my $subject  = Encode::decode("UTF-8",'标题');
+my $content  = Encode::decode("UTF-8",'测试');
+my $filename = Encode::decode("UTF-8",'附件.txt');
+
 diag 'test without attachments' if $ENV{TEST_VERBOSE};
 
 {
@@ -19,13 +21,13 @@ diag 'test without attachments' if $ENV{TEST_VERBOSE};
     $m->form_name('TicketModify');
     $m->submit_form(
         form_number => 3,
-        fields      => { Subject => '标题', Content => '测试' },
+        fields      => { Subject => $subject, Content => $content },
     );
     $m->content_like( qr/Ticket \d+ created/i, 'created the ticket' );
     $m->follow_link_ok( { text => 'with headers' },
         '-> /Ticket/Attachment/WithHeaders/...' );
-    $m->content_contains( '标题', 'has subject 标题' );
-    $m->content_contains( '测试', 'has content 测试' );
+    $m->content_contains( $subject, "has subject $subject" );
+    $m->content_contains( $content, "has content $content" );
 
     my ( $id ) = $m->uri =~ /(\d+)$/;
     ok( $id, 'found attachment id' );
@@ -35,8 +37,8 @@ diag 'test without attachments' if $ENV{TEST_VERBOSE};
     ok( $attachment->SetHeader( 'X-RT-Original-Encoding' => 'gbk' ),
         'set original encoding to gbk' );
     $m->get( $m->uri );
-    $m->content_contains( '标题', 'has subject 标题' );
-    $m->content_contains( '测试', 'has content 测试' );
+    $m->content_contains( $subject, "has subject $subject" );
+    $m->content_contains( $content, "has content $content" );
 }
 
 diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
@@ -44,10 +46,10 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
 {
 
     my $file =
-      File::Spec->catfile( RT::Test->temp_directory, Encode::encode("UTF-8",'附件.txt') );
+      File::Spec->catfile( RT::Test->temp_directory, Encode::encode("UTF-8",$filename) );
     open( my $fh, '>', $file ) or die $!;
     binmode $fh, ':utf8';
-    print $fh '附件';
+    print $fh $filename;
     close $fh;
 
     $m->get_ok( $baseurl . '/Ticket/Create.html?Queue=1' );
@@ -55,17 +57,17 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
     $m->form_name('TicketModify');
     $m->submit_form(
         form_number => 3,
-        fields => { Subject => '标题', Content => '测试', Attach => $file },
+        fields => { Subject => $subject, Content => $content, Attach => $file },
     );
     $m->content_like( qr/Ticket \d+ created/i, 'created the ticket' );
-    $m->content_contains( '附件.txt', 'attached filename' );
-    $m->content_lacks( Encode::encode("UTF-8",'附件.txt'), 'no double encoded attached filename' );
+    $m->content_contains( $filename, 'attached filename' );
+    $m->content_lacks( Encode::encode("UTF-8",$filename), 'no double encoded attached filename' );
     $m->follow_link_ok( { text => 'with headers' },
         '-> /Ticket/Attachment/WithHeaders/...' );
 
     # subject is in the parent attachment, so there is no 标题
-    $m->content_lacks( '标题', 'does not have content 标题' );
-    $m->content_contains( '测试', 'has content 测试' );
+    $m->content_lacks( $subject, "does not have content $subject" );
+    $m->content_contains( $content, "has content $content" );
 
     my ( $id ) = $m->uri =~ /(\d+)$/;
     ok( $id, 'found attachment id' );
@@ -75,15 +77,15 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
     ok( $attachment->SetHeader( 'X-RT-Original-Encoding' => 'gbk' ),
         'set original encoding to gbk' );
     $m->get( $m->uri );
-    $m->content_lacks( '标题', 'does not have content 标题' );
-    $m->content_contains( '测试', 'has content 测试' );
+    $m->content_lacks( $subject, "does not have content $subject" );
+    $m->content_contains( $content, "has content $content" );
 
 
     $m->back;
     $m->back;
-    $m->follow_link_ok( { text => 'Download 附件.txt' },
+    $m->follow_link_ok( { text => "Download $filename" },
         '-> /Ticket/Attachment/...' );
-    $m->content_contains( '附件', 'has content 附件' );
+    $m->content_contains( $filename, "has file content $filename" );
 
     ( $id ) = $m->uri =~ /(\d+)\D+$/;
     ok( $id, 'found attachment id' );
@@ -94,7 +96,7 @@ diag 'test with attachemnts' if $ENV{TEST_VERBOSE};
     ok( $attachment->SetHeader( 'X-RT-Original-Encoding' => 'gbk' ),
         'set original encoding to gbk' );
     $m->get( $m->uri );
-    $m->content_contains( '附件', 'has content 附件' );
+    $m->content_contains( $filename, "has content $filename" );
 
     unlink $file;
 }
diff --git a/t/web/helpers-http-cache-headers.t b/t/web/helpers-http-cache-headers.t
index 420a02d..3b94b07 100644
--- a/t/web/helpers-http-cache-headers.t
+++ b/t/web/helpers-http-cache-headers.t
@@ -1,6 +1,5 @@
 use strict;
 use warnings;
-use utf8;
 
 # trs: I'd write a quick t/web/caching-headers.t file which loops the available
 #      endpoints checking for the right headers.
diff --git a/t/web/html_template.t b/t/web/html_template.t
index 3dbda63..53bace9 100644
--- a/t/web/html_template.t
+++ b/t/web/html_template.t
@@ -2,16 +2,17 @@
 use strict;
 use warnings;
 
-use RT::Test tests => 19;
+use RT::Test tests => undef;
 use Encode;
 my ( $baseurl, $m ) = RT::Test->started_ok;
 ok $m->login, 'logged in as root';
 
-use utf8;
-
 diag('make Autoreply template a html one and add utf8 chars')
   if $ENV{TEST_VERBOSE};
 
+my $template = Encode::decode("UTF-8", "你好 éèà€");
+my $subject  = Encode::decode("UTF-8", "标题");
+my $content  = Encode::decode("UTF-8", "测试");
 {
     $m->follow_link_ok( { id => 'tools-config-global-templates' },     '-> Templates' );
     $m->follow_link_ok( { text => 'Autoreply' },     '-> Autoreply' );
@@ -19,20 +20,20 @@ diag('make Autoreply template a html one and add utf8 chars')
     $m->submit_form(
         form_name => 'ModifyTemplate',
         fields => {
-            Content => <<'EOF',
-Subject: AutoReply: {$Ticket->Subject}
+            Content => <<EOF,
+Subject: AutoReply: {\$Ticket->Subject}
 Content-Type: text/html
 
-你好 éèà€
-{$Ticket->Subject}
+$template
+{\$Ticket->Subject}
 -------------------------------------------------------------------------
-{$Transaction->Content()}
+{\$Transaction->Content()}
 
 EOF
         },
     );
     $m->content_like( qr/Content updated/, 'content is changed' );
-    $m->content_contains( '你好', 'content is really updated' );
+    $m->content_contains( $template, 'content is really updated' );
 }
 
 diag('create a ticket to see the autoreply mail') if $ENV{TEST_VERBOSE};
@@ -42,17 +43,16 @@ diag('create a ticket to see the autoreply mail') if $ENV{TEST_VERBOSE};
 
     $m->submit_form(
         form_name => 'TicketCreate',
-        fields      => { Subject => '标题', Content => '<h1>测试</h1>',
+        fields      => { Subject => $subject, Content => "<h1>$content</h1>",
         ContentType => 'text/html' },
     );
     $m->content_like( qr/Ticket \d+ created/i, 'created the ticket' );
     $m->follow_link( text => 'Show' );
-    $m->content_contains( '你好',    'html has 你好' );
-    $m->content_contains( 'éèà€', 'html has éèà€' );
-    $m->content_contains( '标题',
-        'html has ticket subject 标题' );
-    $m->content_contains( '<h1>测试</h1>',
-        'html has ticket html content 测试' );
+    $m->content_contains( $template, "html has $template" );
+    $m->content_contains( $subject,
+        "html has ticket subject $subject" );
+    $m->content_contains( "<h1>$content</h1>",
+        "html has ticket html content $content" );
 }
 
 diag('test real mail outgoing') if $ENV{TEST_VERBOSE};
@@ -62,10 +62,11 @@ diag('test real mail outgoing') if $ENV{TEST_VERBOSE};
     # $mail is utf8 encoded
     my ($mail) = RT::Test->fetch_caught_mails;
     $mail = Encode::decode("UTF-8", $mail );
-    like( $mail, qr/你好.*你好/s,    'mail has éèà€' );
-    like( $mail, qr/éèà€.*éèà€/s, 'mail has éèà€' );
-    like( $mail, qr/标题.*标题/s,    'mail has ticket subject 标题' );
-    like( $mail, qr/测试.*测试/s,    'mail has ticket content 测试' );
-    like( $mail, qr!<h1>测试</h1>!,    'mail has ticket html content 测试' );
+    like( $mail, qr/$template.*$template/s, 'mail has template content $template twice' );
+    like( $mail, qr/$subject.*$subject/s,   'mail has ticket subject $sujbect twice' );
+    like( $mail, qr/$content.*$content/s,   'mail has ticket content $content twice' );
+    like( $mail, qr!<h1>$content</h1>!,     'mail has ticket html content <h1>$content</h1>' );
 }
 
+undef $m;
+done_testing;
diff --git a/t/web/offline_utf8.t b/t/web/offline_utf8.t
index c317a46..508c361 100644
--- a/t/web/offline_utf8.t
+++ b/t/web/offline_utf8.t
@@ -2,14 +2,13 @@ use strict;
 use warnings;
 
 use RT::Test tests => 9;
-use utf8;
 
 use Encode;
 
 use RT::Ticket;
 my $file = File::Spec->catfile( RT::Test->temp_directory, 'template' );
 open my $fh, '>', $file or die $!;
-my $template = <<EOF;
+my $template = Encode::decode("UTF-8",<<EOF);
 ===Create-Ticket: ticket1
 Queue: General
 Subject: 标题
@@ -19,7 +18,7 @@ Content:
 ENDOFCONTENT
 EOF
 
-print $fh $template;
+print $fh Encode::encode("UTF-8",$template);
 close $fh;
 
 my ( $url, $m ) = RT::Test->started_ok;
@@ -33,7 +32,7 @@ $m->submit_form(
     button    => 'Parse',
 );
 
-$m->content_contains( '这是正文', 'content is parsed right' );
+$m->content_contains( Encode::decode("UTF-8",'这是正文'), 'content is parsed right' );
 
 $m->submit_form(
     form_name => 'TicketUpdate',
@@ -48,9 +47,9 @@ my ( $ticket_id ) = $m->content =~ /Ticket (\d+) created/;
 
 my $ticket = RT::Ticket->new( RT->SystemUser );
 $ticket->Load( $ticket_id );
-is( $ticket->Subject, '标题', 'subject in $ticket is right' );
+is( $ticket->Subject, Encode::decode("UTF-8",'标题'), 'subject in $ticket is right' );
 
 $m->goto_ticket($ticket_id);
-$m->content_contains( '这是正文',
+$m->content_contains( Encode::decode("UTF-8",'这是正文'),
     'content is right in ticket display page' );
 
diff --git a/t/web/ticket_txn_subject.t b/t/web/ticket_txn_subject.t
index 5537023..a43f05d 100644
--- a/t/web/ticket_txn_subject.t
+++ b/t/web/ticket_txn_subject.t
@@ -1,6 +1,5 @@
 use strict;
 use warnings;
-use utf8;
 
 use RT::Test tests => undef;
 
@@ -14,10 +13,10 @@ diag "create a ticket via the API";
     my $ticket = RT::Ticket->new( RT->SystemUser );
     my ($id, $txn, $msg) = $ticket->Create(
         Queue => 'General',
-        Subject => 'bad subject‽',
+        Subject => Encode::decode("UTF-8",'bad subject‽'),
     );
     ok $id, 'created a ticket #'. $id or diag "error: $msg";
-    is $ticket->Subject, 'bad subject‽', 'correct subject';
+    is $ticket->Subject, Encode::decode("UTF-8",'bad subject‽'), 'correct subject';
     push @tickets, $id;
 }
 
@@ -29,10 +28,10 @@ diag "create a ticket via the web";
     }, 'create ticket in Queue');
     $m->submit_form_ok({
         with_fields => {
-            Subject => 'bad subject #2‽',
+            Subject => Encode::decode("UTF-8",'bad subject #2‽'),
         },
     }, 'create ticket');
-    $m->content_contains('bad subject #2‽', 'correct subject');
+    $m->content_contains(Encode::decode("UTF-8",'bad subject #2‽'), 'correct subject');
     push @tickets, 2;
 }
 
@@ -58,12 +57,12 @@ for my $tid (@tickets) {
         $m->follow_link_ok({ id => 'page-actions-reply' }, "Actions -> Reply");
         $m->submit_form_ok({
             with_fields => {
-                UpdateSubject => 'bad subject‽ without attachment',
+                UpdateSubject => Encode::decode("UTF-8",'bad subject‽ without attachment'),
                 UpdateContent => 'testing unicode txn subjects',
             },
             button => 'SubmitTicket',
         }, 'submit reply');
-        $m->content_contains('bad subject‽ without attachment', "found txn subject");
+        $m->content_contains(Encode::decode("UTF-8",'bad subject‽ without attachment'), "found txn subject");
     }
 
     diag "add a reply which adds to the subject with an attachment";
@@ -72,13 +71,13 @@ for my $tid (@tickets) {
         $m->follow_link_ok({ id => 'page-actions-reply' }, "Actions -> Reply");
         $m->submit_form_ok({
             with_fields => {
-                UpdateSubject => 'bad subject‽ with attachment',
+                UpdateSubject => Encode::decode("UTF-8",'bad subject‽ with attachment'),
                 UpdateContent => 'testing unicode txn subjects',
                 Attach => RT::Test::get_relocatable_file('bpslogo.png', '..', 'data'),
             },
             button => 'SubmitTicket',
         }, 'submit reply');
-        $m->content_contains('bad subject‽ with attachment', "found txn subject");
+        $m->content_contains(Encode::decode("UTF-8",'bad subject‽ with attachment'), "found txn subject");
     }
 }
 
diff --git a/t/web/user_update.t b/t/web/user_update.t
index c0e9e52..54139d7 100644
--- a/t/web/user_update.t
+++ b/t/web/user_update.t
@@ -1,6 +1,5 @@
 use strict;
 use warnings;
-use utf8;
 use RT::Test tests => undef;
 
 my ( $url, $m ) = RT::Test->started_ok;
@@ -10,7 +9,7 @@ $m->follow_link_ok({text => 'About me'});
 $m->submit_form_ok({ with_fields => { Lang => 'ja'} },
                "Change to Japanese");
 $m->text_contains("Lang changed from (no value) to 'ja'");
-$m->text_contains("実名", "Page content is japanese");
+$m->text_contains(Encode::decode("UTF-8","実名"), "Page content is japanese");
 
 # we only changed one field, and it wasn't the default, so this feedback is
 # spurious and annoying
@@ -22,7 +21,7 @@ $m->submit_form_ok({ with_fields => { Lang => 'en_us'} },
 
 # This message shows up in Japanese
 # $m->text_contains("Lang changed from 'ja' to 'en_us'");
-$m->text_contains("Langは「'ja'」から「'en_us'」に変更されました");
+$m->text_contains(Encode::decode("UTF-8","Langは「'ja'」から「'en_us'」に変更されました"));
 $m->text_contains("Real Name", "Page content is english");
 
 # Check for a lack of spurious updates
@@ -32,12 +31,11 @@ $m->content_lacks("That is already the current value");
 $m->submit_form_ok({ with_fields => { Lang => 'ja'} },
                    "Back briefly to Japanese");
 $m->text_contains("Lang changed from 'en_us' to 'ja'");
-$m->text_contains("実名", "Page content is japanese");
+$m->text_contains(Encode::decode("UTF-8","実名"), "Page content is japanese");
 $m->submit_form_ok({ with_fields => { Lang => ''} },
                    "And set to the default");
-$m->text_contains("Langは「'ja'」から「(値なし)」に変更されました");
+$m->text_contains(Encode::decode("UTF-8","Langは「'ja'」から「(値なし)」に変更されました"));
 $m->text_contains("Real Name", "Page content is english");
 
 undef $m;
-
 done_testing;

commit 9fc8d082ce8db1a1425926474fa9da9288a98bb7
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:55:25 2014 -0400

    Dashboard: decode bytes in query parameters into characters
    
    The dashboard mailer code replicates many of the original steps of Mason
    parameter parsing -- but omitted the important step of decoding the
    bytes once they had been un-URI-encoded.

diff --git a/lib/RT/Dashboard/Mailer.pm b/lib/RT/Dashboard/Mailer.pm
index 92f1a0d..038cf45 100644
--- a/lib/RT/Dashboard/Mailer.pm
+++ b/lib/RT/Dashboard/Mailer.pm
@@ -553,6 +553,9 @@ sub GetResource {
 
         for ($k, $v) { s/%(..)/chr hex $1/ge }
 
+        # Decode from bytes to characters
+        $_ = Encode::decode( "UTF-8", $_ ) for $k, $v;
+
         # no value yet, simple key=value
         if (!exists $args{$k}) {
             $args{$k} = $v;

commit 53dbebc8c0b197ab5555cfb2a65eefecd0b0af6b
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:56:23 2014 -0400

    Tests: WWW::Mechanize correctly returns characters now
    
    While there ay have been bugs surrounding encodings in LWP or
    WWW::Mechanize previously, the ->content method correctly returns
    characters and not bytes for all modern versions.  Remove the explicit
    adjustment.

diff --git a/t/i18n/default.t b/t/i18n/default.t
index 8d8f98b..d98828f 100644
--- a/t/i18n/default.t
+++ b/t/i18n/default.t
@@ -13,7 +13,6 @@ $m->content_contains('<html lang="en">');
 
 $m->add_header('Accept-Language' => 'zh-tw,zh;q=0.8,en-gb;q=0.5,en;q=0.3');
 $m->get_ok('/');
-Encode::_utf8_on($m->{content});
 $m->title_is( Encode::decode("UTF-8",'登入'),
               'Page title properly translated to chinese');
 $m->content_contains( Encode::decode("UTF-8",'密碼'),

commit 4522c09a100c8c9c034265b1c1f0100f04cb5714
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:57:15 2014 -0400

    _utf8_on in EncodeToMIME is needless and incorrect; remove it
    
    66930fd8 switched from an explicit _utf8_off to an explicit _utf8_on, in
    an attempt to switch from splitting on bytes to splitting on characters.
    However, the "UTF8" flag does not magically determine if a string is
    bytes or characters.  Instead, only consistency in calling convention
    can do so.  All callsites of RT::Interface::Email::EncodeToMIME and
    RT::Action::SendEmail::MIMEEncodeString now pass character strings; all
    that _utf8_on can do is incorrectly "decode" those strings as UTF-8 if
    they happen to not have the "UTF8" flag set.

diff --git a/lib/RT/Interface/Email.pm b/lib/RT/Interface/Email.pm
index bd42d3d..6afb1c1 100644
--- a/lib/RT/Interface/Email.pm
+++ b/lib/RT/Interface/Email.pm
@@ -987,9 +987,6 @@ sub EncodeToMIME {
 
     $value =~ s/\s+$//;
 
-    # we need perl string to split thing char by char
-    Encode::_utf8_on($value) unless Encode::is_utf8($value);
-
     my ( $tmp, @chunks ) = ( '', () );
     while ( length $value ) {
         my $char = substr( $value, 0, 1, '' );

commit ed465cf64cfdbf2468dd84a8bce63212b1632841
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 13:57:41 2014 -0400

    Move comment from PreprocessTimeUpdates to DecodeArgs, where it belongs
    
    c95221e4 moved this comment from html/autohandler, but associated it
    with the code it was above rather than the code above it.  Move it to
    where it belongs, and update it slightly.

diff --git a/lib/RT/Interface/Web.pm b/lib/RT/Interface/Web.pm
index a6f8a56..c2d0926 100644
--- a/lib/RT/Interface/Web.pm
+++ b/lib/RT/Interface/Web.pm
@@ -1121,6 +1121,14 @@ sub StripContent {
 sub DecodeARGS {
     my $ARGS = shift;
 
+    # Later in the code we use
+    # $m->comp( { base_comp => $m->request_comp }, $m->fetch_next, %ARGS );
+    # instead of $m->call_next to avoid problems with UTF8 keys in
+    # arguments.  Specifically, the call_next method pass through
+    # original arguments, which are still the encoded bytes, not
+    # characters.  "{ base_comp => $m->request_comp }" is copied from
+    # mason's source to get the same results as we get from call_next
+    # method; this feature is not documented.
     %{$ARGS} = map {
 
         # if they've passed multiple values, they'll be an array. if they've
@@ -1143,17 +1151,6 @@ sub DecodeARGS {
 sub PreprocessTimeUpdates {
     my $ARGS = shift;
 
-    # Later in the code we use
-    # $m->comp( { base_comp => $m->request_comp }, $m->fetch_next, %ARGS );
-    # instead of $m->call_next to avoid problems with UTF8 keys in arguments.
-    # The call_next method pass through original arguments and if you have
-    # an argument with unicode key then in a next component you'll get two
-    # records in the args hash: one with key without UTF8 flag and another
-    # with the flag, which may result into errors. "{ base_comp => $m->request_comp }"
-    # is copied from mason's source to get the same results as we get from
-    # call_next method, this feature is not documented, so we just leave it
-    # here to avoid possible side effects.
-
     # This code canonicalizes time inputs in hours into minutes
     foreach my $field ( keys %$ARGS ) {
         next unless $field =~ /^(.*)-TimeUnits$/i && $ARGS->{$1};

commit 2eca7798cc9fb3c15f64a110432d2101ff69a994
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 14:00:22 2014 -0400

    Always decode data in %ARGS as UTF-8 in DecodeArgs
    
    There is no need to check is_utf8 on the arguments passed to DecodeArgs,
    as it is the first and only line of decoding of arguments over HTTP.

diff --git a/lib/RT/Interface/Web.pm b/lib/RT/Interface/Web.pm
index c2d0926..64e53c4 100644
--- a/lib/RT/Interface/Web.pm
+++ b/lib/RT/Interface/Web.pm
@@ -1135,15 +1135,11 @@ sub DecodeARGS {
         # passed just one, a scalar whatever they are, mark them as utf8
         my $type = ref($_);
         ( !$type )
-            ? Encode::is_utf8($_)
-                ? $_
-                : Encode::decode( 'UTF-8' => $_, Encode::FB_PERLQQ )
+            ? Encode::decode( 'UTF-8', $_, Encode::FB_PERLQQ )
             : ( $type eq 'ARRAY' )
-            ? [ map { ( ref($_) or Encode::is_utf8($_) ) ? $_ : Encode::decode( 'UTF-8' => $_, Encode::FB_PERLQQ ) }
-                @$_ ]
+            ? [ map { ref($_) ? $_ : Encode::decode( 'UTF-8', $_, Encode::FB_PERLQQ ) } @$_ ]
             : ( $type eq 'HASH' )
-            ? { map { ( ref($_) or Encode::is_utf8($_) ) ? $_ : Encode::decode( 'UTF-8' => $_, Encode::FB_PERLQQ ) }
-                %$_ }
+            ? { map { ref($_) ? $_ : Encode::decode( 'UTF-8', $_, Encode::FB_PERLQQ ) } %$_ }
             : $_
     } %$ARGS;
 }

commit 0afbecaa9759e20d7bbfcfebb3c4e1ac4ab72c25
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 14:05:39 2014 -0400

    Add RT::Util::assert_bytes checks to _EncodeLOB and _DecodeLOB
    
    929b4231 specifically documented _EncodeLOB and _DecodeLOB to take
    bytes; enforce that by checking the arguments they are passed using
    RT::Util::assert_bytes.

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 8ee0ff8..8536673 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -754,6 +754,8 @@ sub _EncodeLOB {
 
     my $ContentEncoding = 'none';
 
+    RT::Util::assert_bytes( $Body );
+
     #get the max attachment length from RT
     my $MaxSize = RT->Config->Get('MaxAttachmentSize');
 
@@ -800,17 +802,13 @@ sub _EncodeLOB {
 
     # if we need to mimencode the attachment
     if ( $ContentEncoding eq 'base64' ) {
-
         # base64 encode the attachment
-        Encode::_utf8_off($Body);
         $Body = MIME::Base64::encode_base64($Body);
 
     } elsif ($ContentEncoding eq 'quoted-printable') {
-        Encode::_utf8_off($Body);
         $Body = MIME::QuotedPrint::encode($Body);
     }
 
-
     return ($ContentEncoding, $Body, $MIMEType, $Filename );
 }
 
@@ -847,6 +845,8 @@ sub _DecodeLOB {
     my $ContentEncoding = shift || 'none';
     my $Content         = shift;
 
+    RT::Util::assert_bytes( $Content );
+
     if ( $ContentEncoding eq 'base64' ) {
         $Content = MIME::Base64::decode_base64($Content);
     }
@@ -863,7 +863,7 @@ sub _DecodeLOB {
         my $charset = RT::I18N::_FindOrGuessCharset($entity);
         $charset = 'utf-8' if not $charset or not Encode::find_encoding($charset);
 
-        $Content = Encode::decode($charset,$Content,Encode::FB_PERLQQ) unless Encode::is_utf8($Content);
+        $Content = Encode::decode($charset,$Content,Encode::FB_PERLQQ);
     }
     return ($Content);
 }

commit a382b5002d7f3c0d898d910510f37b6c46ce8984
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 14:08:01 2014 -0400

    Update POD and comments to be clearer about characters vs bytes

diff --git a/lib/RT/Action/SendEmail.pm b/lib/RT/Action/SendEmail.pm
index 500ab57..3613a88 100644
--- a/lib/RT/Action/SendEmail.pm
+++ b/lib/RT/Action/SendEmail.pm
@@ -1085,7 +1085,8 @@ sub PseudoReference {
 
 =head2 SetHeaderAsEncoding($field_name, $charset_encoding)
 
-This routine converts the field into specified charset encoding.
+This routine converts the field into specified charset encoding, then
+applies the MIME-Header transfer encoding.
 
 =cut
 
diff --git a/lib/RT/I18N.pm b/lib/RT/I18N.pm
index f702b3a..3a45c98 100644
--- a/lib/RT/I18N.pm
+++ b/lib/RT/I18N.pm
@@ -261,14 +261,8 @@ sub SetMIMEEntityToEncoding {
 =head2 DecodeMIMEWordsToUTF8 $raw
 
 An utility method which mimics MIME::Words::decode_mimewords, but only
-limited functionality.  This function returns an utf-8 string.
-
-It returns the decoded string, or the original string if it's not
-encoded.  Since the subroutine converts specified string into utf-8
-charset, it should not alter a subject written in English.
-
-Why not use MIME::Words directly?  Because it fails in RT when I
-tried.  Maybe it's ok now.
+limited functionality.  Despite its name, this function returns the
+bytes of the string, in UTF-8.
 
 =cut
 
diff --git a/lib/RT/Template.pm b/lib/RT/Template.pm
index 6d92eb9..a6c0f7d 100644
--- a/lib/RT/Template.pm
+++ b/lib/RT/Template.pm
@@ -307,10 +307,9 @@ sub IsEmpty {
 Returns L<MIME::Entity> object parsed using L</Parse> method. Returns
 undef if last call to L</Parse> failed or never be called.
 
-Note that content of the template is UTF-8, but L<MIME::Parser> is not
-good at handling it and all data of the entity should be treated as
-octets and converted to perl strings using Encode::decode_utf8 or
-something else.
+Note that content of the template is characters, but the contents of all
+L<MIME::Entity> objects (including the one returned by this function,
+are bytes in UTF-8.
 
 =cut
 
diff --git a/t/web/rest-non-ascii-subject.t b/t/web/rest-non-ascii-subject.t
index bc0b53e..42e05a9 100644
--- a/t/web/rest-non-ascii-subject.t
+++ b/t/web/rest-non-ascii-subject.t
@@ -4,7 +4,6 @@ use warnings;
 use RT::Test tests => 9;
 
 use Encode;
-# \x{XX} where XX is less than 255 is not treated as unicode code point
 my $subject = Encode::decode('latin1', "Sujet accentu\x{e9}");
 my $text = Encode::decode('latin1', "Contenu accentu\x{e9}");
 

commit dbb9efedca65af28fa22db8d9cb23a0f749f8d79
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 14:08:41 2014 -0400

    Remove an unreachable line

diff --git a/lib/RT/Interface/Web/Handler.pm b/lib/RT/Interface/Web/Handler.pm
index 0434e7c..202ddc8 100644
--- a/lib/RT/Interface/Web/Handler.pm
+++ b/lib/RT/Interface/Web/Handler.pm
@@ -329,7 +329,6 @@ sub _psgi_response_cb {
                          return '';
                      }
                      return utf8::is_utf8($_[0]) ? Encode::encode( "UTF-8", $_[0]) : $_[0];
-                     return $_[0];
                  };
              });
     }

commit 81132aa503165ccd6cbd0b758bddd59be06a24dc
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 15:12:10 2014 -0400

    TSV need not explicitly encode as UTF-8; all output is UTF-8 encoded
    
    RT::Interface::Web::Handler ensures that any code which contains
    characters is encoded in UTF-8 before being sent to the browser.  As
    such, the explicit encoding here is unnecessary.

diff --git a/share/html/Search/Results.tsv b/share/html/Search/Results.tsv
index c843614..0c57711 100644
--- a/share/html/Search/Results.tsv
+++ b/share/html/Search/Results.tsv
@@ -70,7 +70,7 @@ my $col_entry = sub {
     delete $col->{title}
         if $col->{title} and $col->{title} =~ /^\s*#\s*$/;
     return {
-        header => Encode::encode( "UTF-8", loc($col->{title} || $col->{attribute})),
+        header => loc($col->{title} || $col->{attribute}),
         map    => $m->comp(
             "/Elements/ColumnMap",
             Name  => $col->{attribute},
@@ -127,7 +127,7 @@ while (my $row = $Tickets->Next) {
             # remove tabs from all field values, they screw up the tsv
             $val = '' unless defined $val;
             $val =~ s/(?:\n|\r)//g; $val =~ s{\t}{    }g;
-            Encode::encode( "UTF-8", $val);
+            $val;
         } @$col)."\n");
     }
 }

commit 44ec57197910c2bac9047a688d408bdc77276394
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 15:38:41 2014 -0400

    Move "use Encode" calls to one central location

diff --git a/lib/RT.pm b/lib/RT.pm
index e7da1a9..140d43e 100644
--- a/lib/RT.pm
+++ b/lib/RT.pm
@@ -52,6 +52,7 @@ use warnings;
 package RT;
 
 
+use Encode ();
 use File::Spec ();
 use Cwd ();
 
diff --git a/lib/RT/I18N.pm b/lib/RT/I18N.pm
index 3a45c98..11cd5f1 100644
--- a/lib/RT/I18N.pm
+++ b/lib/RT/I18N.pm
@@ -62,7 +62,6 @@ use Locale::Maketext 1.04;
 use Locale::Maketext::Lexicon 0.25;
 use base 'Locale::Maketext::Fuzzy';
 
-use Encode;
 use MIME::Entity;
 use MIME::Head;
 use File::Glob;
diff --git a/lib/RT/Interface/Web.pm b/lib/RT/Interface/Web.pm
index 64e53c4..3d8cd99 100644
--- a/lib/RT/Interface/Web.pm
+++ b/lib/RT/Interface/Web.pm
@@ -68,7 +68,6 @@ use URI qw();
 use RT::Interface::Web::Menu;
 use RT::Interface::Web::Session;
 use Digest::MD5 ();
-use Encode qw();
 use List::MoreUtils qw();
 use JSON qw();
 
diff --git a/lib/RT/Interface/Web/Handler.pm b/lib/RT/Interface/Web/Handler.pm
index 202ddc8..3c9e2bb 100644
--- a/lib/RT/Interface/Web/Handler.pm
+++ b/lib/RT/Interface/Web/Handler.pm
@@ -251,7 +251,6 @@ use CGI::Emulate::PSGI;
 use Plack::Request;
 use Plack::Response;
 use Plack::Util;
-use Encode;
 
 sub PSGIApp {
     my $self = shift;
diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 8536673..f9602e4 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -70,7 +70,6 @@ use warnings;
 use RT::Date;
 use RT::User;
 use RT::Attributes;
-use Encode qw();
 
 our $_TABLE_ATTR = { };
 use base RT->Config->Get('RecordBaseClass');
diff --git a/lib/RT/User.pm b/lib/RT/User.pm
index 50016cc..22f9119 100644
--- a/lib/RT/User.pm
+++ b/lib/RT/User.pm
@@ -81,7 +81,6 @@ use Digest::MD5;
 use RT::Principals;
 use RT::ACE;
 use RT::Interface::Email;
-use Encode;
 use Text::Password::Pronounceable;
 
 sub _OverlayAccessible {
diff --git a/share/html/NoAuth/iCal/dhandler b/share/html/NoAuth/iCal/dhandler
index 4eaa254..46c2729 100644
--- a/share/html/NoAuth/iCal/dhandler
+++ b/share/html/NoAuth/iCal/dhandler
@@ -48,7 +48,6 @@
 <%init>
 use Data::ICal;
 use Data::ICal::Entry::Event;
-use Encode ();
 
 my $path = $m->dhandler_arg;
 
diff --git a/share/html/Search/Elements/ResultsRSSView b/share/html/Search/Elements/ResultsRSSView
index 41e200f..a453a86 100644
--- a/share/html/Search/Elements/ResultsRSSView
+++ b/share/html/Search/Elements/ResultsRSSView
@@ -46,8 +46,6 @@
 %#
 %# END BPS TAGGED BLOCK }}}
 <%INIT>
-use Encode ();
-
 my $old_current_user;
 
 if ( $m->request_comp->path =~ RT->Config->Get('WebNoAuthRegex') ) {
diff --git a/share/html/Tools/Offline.html b/share/html/Tools/Offline.html
index 507ca17..de49e00 100644
--- a/share/html/Tools/Offline.html
+++ b/share/html/Tools/Offline.html
@@ -114,7 +114,6 @@ if ($ARGS{'Parse'} && $ARGS{'Template'}) {
 	    $template .= $buffer;
     }
     my $encode = RT::I18N::_GuessCharset( $template );
-    require Encode;
     $template = Encode::decode( $encode, $template );
     $template =~ s/\r\n/\n/gs;
     $action->Parse(Content => $template, Queue => $qname, Requestor => $requestoraddress);
diff --git a/t/00-mason-syntax.t b/t/00-mason-syntax.t
index e4ea007..ac0da0d 100644
--- a/t/00-mason-syntax.t
+++ b/t/00-mason-syntax.t
@@ -20,7 +20,6 @@ use HTML::Mason;
 use HTML::Mason::Compiler;
 use HTML::Mason::Compiler::ToObject;
 BEGIN { require RT::Test; }
-use Encode;
 
 sub compile_file {
     my $file = shift;
diff --git a/t/api/attachment.t b/t/api/attachment.t
index 4d607e6..52e3c3f 100644
--- a/t/api/attachment.t
+++ b/t/api/attachment.t
@@ -58,7 +58,6 @@ is ($#headers, 2, "testing a bunch of singline multiple headers" );
     my $mime = $attachment->ContentAsMIME;
     like( $mime->head->get('Content-Type'),
         qr/charset="iso-8859-1"/, 'content type of ContentAsMIME is original' );
-    require Encode;
     is(
         Encode::decode( 'iso-8859-1', $mime->stringify_body ),
         Encode::decode( 'UTF-8',      "HÃ¥vard\n" ),
diff --git a/t/api/canonical_charset.t b/t/api/canonical_charset.t
index 8658723..86c3e97 100644
--- a/t/api/canonical_charset.t
+++ b/t/api/canonical_charset.t
@@ -3,7 +3,6 @@ use strict;
 
 use RT::Test nodata => 1, tests => 11;
 use RT::I18N;
-use Encode;
 
 my %map = (
     'euc-cn'  => 'gbk',
diff --git a/t/api/i18n_guess.t b/t/api/i18n_guess.t
index 956cb15..a64b295 100644
--- a/t/api/i18n_guess.t
+++ b/t/api/i18n_guess.t
@@ -4,8 +4,6 @@ use warnings;
 
 use RT::Test tests => 16;
 
-use Encode qw(encode);
-
 use constant HAS_ENCODE_GUESS => do { local $@; eval { require Encode::Guess; 1 } };
 use constant HAS_ENCODE_DETECT => do { local $@; eval { require Encode::Detect::Detector; 1 } };
 
diff --git a/t/api/password-types.t b/t/api/password-types.t
index 22b6c91..4cb6342 100644
--- a/t/api/password-types.t
+++ b/t/api/password-types.t
@@ -3,7 +3,6 @@ use warnings;
 
 use RT::Test;
 use Digest::MD5;
-use Encode;
 
 my $default = "sha512";
 
diff --git a/t/mail/charsets-outgoing.t b/t/mail/charsets-outgoing.t
index 165f823..8727213 100644
--- a/t/mail/charsets-outgoing.t
+++ b/t/mail/charsets-outgoing.t
@@ -1,6 +1,5 @@
 use strict;
 use warnings;
-use Encode;
 
 use RT::Test tests => 78;
 
diff --git a/t/mail/header-characters.t b/t/mail/header-characters.t
index 63ad525..004ba85 100644
--- a/t/mail/header-characters.t
+++ b/t/mail/header-characters.t
@@ -3,7 +3,6 @@ use warnings;
 
 use RT::Test tests => 12;
 use Test::Warn;
-use Encode;
 
 my ($baseurl, $m) = RT::Test->started_ok;
 
diff --git a/t/mail/sendmail.t b/t/mail/sendmail.t
index acfedce..a749bc3 100644
--- a/t/mail/sendmail.t
+++ b/t/mail/sendmail.t
@@ -5,7 +5,6 @@ use RT::Test tests => undef;
 
 use File::Spec ();
 use Email::Abstract;
-use Encode;
 
 # We're not testing acls here.
 my $everyone = RT::Group->new(RT->SystemUser);
@@ -113,7 +112,7 @@ for my $encoding ('ISO-8859-1', 'UTF-8') {
 
     my $encoded_subject = $mail[0]->head->get("Subject");
     chomp $encoded_subject;
-    my $subject = decode('MIME-Header',$encoded_subject);
+    my $subject = Encode::decode('MIME-Header',$encoded_subject);
     like($subject, qr/Niv\x{e5}er/, Encode::encode("UTF-8", "The subject matches the word - $subject"));
 }
 
diff --git a/t/mail/wrong_mime_charset.t b/t/mail/wrong_mime_charset.t
index 7880149..6bbaca1 100644
--- a/t/mail/wrong_mime_charset.t
+++ b/t/mail/wrong_mime_charset.t
@@ -3,9 +3,8 @@ use warnings;
 use RT::Test nodb => 1, tests => 6;
 
 use_ok('RT::I18N');
-use Encode;
 my $test_string    = Encode::decode("UTF-8", 'À');
-my $encoded_string = encode( 'iso-8859-1', $test_string );
+my $encoded_string = Encode::encode( 'iso-8859-1', $test_string );
 my $mime           = MIME::Entity->build(
     "Subject" => $encoded_string,
     "Data"    => [$encoded_string],
@@ -39,10 +38,10 @@ like(
 "We can't encode something into the wrong encoding without Encode complaining"
 );
 
-my $subject = decode( 'iso-8859-1', $mime->head->get('Subject') );
+my $subject = Encode::decode( 'iso-8859-1', $mime->head->get('Subject') );
 chomp $subject;
 is( $subject, $test_string, 'subject is set to iso-8859-1' );
-my $body = decode( 'iso-8859-1', $mime->stringify_body );
+my $body = Encode::decode( 'iso-8859-1', $mime->stringify_body );
 chomp $body;
 is( $body, $test_string, 'body is set to iso-8859-1' );
 }
diff --git a/t/web/attachment_encoding.t b/t/web/attachment_encoding.t
index fb768dc..f49720e 100644
--- a/t/web/attachment_encoding.t
+++ b/t/web/attachment_encoding.t
@@ -3,7 +3,6 @@ use strict;
 use warnings;
 
 use RT::Test tests => 32;
-use Encode;
 my ( $baseurl, $m ) = RT::Test->started_ok;
 ok $m->login, 'logged in as root';
 
diff --git a/t/web/basic.t b/t/web/basic.t
index 430d8c8..02483b2 100644
--- a/t/web/basic.t
+++ b/t/web/basic.t
@@ -1,7 +1,6 @@
 
 use strict;
 use warnings;
-use Encode;
 
 use RT::Test tests => 23;
 
diff --git a/t/web/compilation_errors.t b/t/web/compilation_errors.t
index 0ae6ead..126d336 100644
--- a/t/web/compilation_errors.t
+++ b/t/web/compilation_errors.t
@@ -15,7 +15,6 @@ BEGIN {
 use HTTP::Request::Common;
 use HTTP::Cookies;
 use LWP;
-use Encode;
 
 my $cookie_jar = HTTP::Cookies->new;
 
diff --git a/t/web/html_template.t b/t/web/html_template.t
index 53bace9..a276455 100644
--- a/t/web/html_template.t
+++ b/t/web/html_template.t
@@ -3,7 +3,6 @@ use strict;
 use warnings;
 
 use RT::Test tests => undef;
-use Encode;
 my ( $baseurl, $m ) = RT::Test->started_ok;
 ok $m->login, 'logged in as root';
 
diff --git a/t/web/offline_messages_utf8.t b/t/web/offline_messages_utf8.t
index 8a54140..4cf6954 100644
--- a/t/web/offline_messages_utf8.t
+++ b/t/web/offline_messages_utf8.t
@@ -2,7 +2,6 @@ use strict;
 use warnings;
 
 use RT::Test tests => 8;
-use Encode;
 use RT::Ticket;
 
 my ( $url, $m ) = RT::Test->started_ok;
diff --git a/t/web/offline_utf8.t b/t/web/offline_utf8.t
index 508c361..aab3049 100644
--- a/t/web/offline_utf8.t
+++ b/t/web/offline_utf8.t
@@ -3,8 +3,6 @@ use warnings;
 
 use RT::Test tests => 9;
 
-use Encode;
-
 use RT::Ticket;
 my $file = File::Spec->catfile( RT::Test->temp_directory, 'template' );
 open my $fh, '>', $file or die $!;
diff --git a/t/web/query_builder.t b/t/web/query_builder.t
index 13cd1b5..3589c38 100644
--- a/t/web/query_builder.t
+++ b/t/web/query_builder.t
@@ -3,7 +3,6 @@ use warnings;
 use HTTP::Request::Common;
 use HTTP::Cookies;
 use LWP;
-use Encode;
 use RT::Test tests => 70;
 
 my $cookie_jar = HTTP::Cookies->new;
diff --git a/t/web/rest-non-ascii-subject.t b/t/web/rest-non-ascii-subject.t
index 42e05a9..0d3e14d 100644
--- a/t/web/rest-non-ascii-subject.t
+++ b/t/web/rest-non-ascii-subject.t
@@ -3,7 +3,6 @@ use strict;
 use warnings;
 use RT::Test tests => 9;
 
-use Encode;
 my $subject = Encode::decode('latin1', "Sujet accentu\x{e9}");
 my $text = Encode::decode('latin1', "Contenu accentu\x{e9}");
 
diff --git a/t/web/ticket-create-utf8.t b/t/web/ticket-create-utf8.t
index bebc57b..107e41d 100644
--- a/t/web/ticket-create-utf8.t
+++ b/t/web/ticket-create-utf8.t
@@ -4,8 +4,6 @@ use warnings;
 
 use RT::Test tests => 43;
 
-use Encode;
-
 my $ru_test = "\x{442}\x{435}\x{441}\x{442}";
 my $ru_support = "\x{43f}\x{43e}\x{434}\x{434}\x{435}\x{440}\x{436}\x{43a}\x{430}";
 

commit 779dc60b94a6c5fd7f863dfacb8df2dec3e1b65c
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 15:40:27 2014 -0400

    Consistent character/byte hygene allows RT to run with DBD::Pg 3.3.0
    
    This does require a version dump of the DBIx::SearchBuilder dependency
    for PostgreSQL installs, as DBIx::SearchBuilder previous to that version
    manually forced the "UTF8" off on all bound values before executing
    statements.  When the character/byte distinction has been intentionally
    made, manually (and unpredictably) applying an additional layer of UTF-8
    encoding is incorrect.

diff --git a/sbin/rt-test-dependencies.in b/sbin/rt-test-dependencies.in
index e7cca1f..679b41d 100644
--- a/sbin/rt-test-dependencies.in
+++ b/sbin/rt-test-dependencies.in
@@ -334,6 +334,7 @@ DBD::Oracle
 .
 
 $deps{'POSTGRESQL'} = [ text_to_hash( << '.') ];
+DBIx::SearchBuilder 1.66
 DBD::Pg 1.43
 .
 
@@ -382,7 +383,6 @@ HTML::Entities
 
 my %AVOID = (
     'DBD::Oracle' => [qw(1.23)],
-    'DBD::Pg' => [qw(3.3.0)],
     'Email::Address' => [qw(1.893 1.894)],
     'Devel::StackTrace' => [qw(1.28 1.29)],
 );

commit c2f0fe0402e7cb60e431164d168e3eda0f185d1a
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 15:42:56 2014 -0400

    Note that HTTP output still incorrectly relies on is_utf8
    
    Currently, any string which has the "UTF8" flag is encoded as UTF-8
    before being sent to the browser.  This requires that any output which
    is binary, or has already been encoded to bytes, _not_ have the flag
    accidentally set.
    
    It also requires that all output character strings have the "UTF8" flag
    enabled; while necessary for codepoints > 255, it is not strictly
    required for codepoints between 127 and 255.  As RT now consistently
    uses Encode::decode() to produce character strings, which sets the
    "UTF8" flag even for characters in that range, this is likely safe.
    
    The most correct fix would be to explicitly flag output that needs to be
    encoded.  However, doing so in a backwards compatible manner is
    extremely difficult; as is_utf8 is unlikely to be incorrect in this
    context, the small potential additional correctness is deemed unworth
    the cost of requiring all external modules to flag their binary (or
    character) output as such.

diff --git a/lib/RT/Interface/Web/Handler.pm b/lib/RT/Interface/Web/Handler.pm
index 3c9e2bb..7cf18d1 100644
--- a/lib/RT/Interface/Web/Handler.pm
+++ b/lib/RT/Interface/Web/Handler.pm
@@ -327,7 +327,11 @@ sub _psgi_response_cb {
                          $cleanup->();
                          return '';
                      }
-                     return utf8::is_utf8($_[0]) ? Encode::encode( "UTF-8", $_[0]) : $_[0];
+                     # XXX: Ideally, responses should flag if they need
+                     # to be encoded, rather than relying on the UTF-8
+                     # flag
+                     return Encode::encode("UTF-8",$_[0]) if utf8::is_utf8($_[0]);
+                     return $_[0];
                  };
              });
     }

commit 302989a2591195ed5f66188c1759c6523f839150
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 15:44:51 2014 -0400

    Comment the logic for database decode_utf8/is_utf8 checking

diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index f9602e4..d534b7d 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -644,12 +644,16 @@ sub __Value {
 
     return undef if (!defined $value);
 
+    # Pg returns character columns as character strings; mysql and
+    # sqlite return them as bytes.  While mysql can be made to return
+    # characters, using the mysql_enable_utf8 flag, the "Content" column
+    # is bytes on mysql and characters on Postgres, making true
+    # consistency impossible.
     if ( $args{'decode_utf8'} ) {
-        if ( !utf8::is_utf8($value) ) {
+        if ( !utf8::is_utf8($value) ) { # mysql/sqlite
             utf8::decode($value);
         }
-    }
-    else {
+    } else {
         if ( utf8::is_utf8($value) ) {
             utf8::encode($value);
         }

commit a07d8113e30f9a0c8fbe99e0b0762d84f5ce2d0f
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 11:16:37 2014 -0400

    Encode characters on their way out of tests
    
    Tests may produce characters (e.g. ok(1, "¡Oy!") ) which need to be
    encoded before being output.  Use the suggested technique from the
    Test::More POD to encode these strings.

diff --git a/lib/RT/Test.pm b/lib/RT/Test.pm
index 6ee1587..e956160 100644
--- a/lib/RT/Test.pm
+++ b/lib/RT/Test.pm
@@ -164,6 +164,8 @@ sub import {
 
     $class->set_config_wrapper;
 
+    $class->encode_output;
+
     my $screen_logger = $RT::Logger->remove( 'screen' );
     require Log::Dispatch::Perl;
     $RT::Logger->add( Log::Dispatch::Perl->new
@@ -417,6 +419,13 @@ sub set_config_wrapper {
     };
 }
 
+sub encode_output {
+    my $builder = Test::More->builder;
+    binmode $builder->output,         ":encoding(utf8)";
+    binmode $builder->failure_output, ":encoding(utf8)";
+    binmode $builder->todo_output,    ":encoding(utf8)";
+}
+
 sub bootstrap_db {
     my $self = shift;
     my %args = @_;
diff --git a/t/mail/sendmail.t b/t/mail/sendmail.t
index a749bc3..56202ad 100644
--- a/t/mail/sendmail.t
+++ b/t/mail/sendmail.t
@@ -113,7 +113,7 @@ for my $encoding ('ISO-8859-1', 'UTF-8') {
     my $encoded_subject = $mail[0]->head->get("Subject");
     chomp $encoded_subject;
     my $subject = Encode::decode('MIME-Header',$encoded_subject);
-    like($subject, qr/Niv\x{e5}er/, Encode::encode("UTF-8", "The subject matches the word - $subject"));
+    like($subject, qr/Niv\x{e5}er/, "The subject matches the word - $subject");
 }
 
 {

commit e96002ae4dadf2125e3ead0e1940cb5df9f4c78a
Author: Alex Vandiver <alexmv at bestpractical.com>
Date:   Fri Aug 8 11:16:53 2014 -0400

    Stop hiding "Wide character in..." warnings
    
    As input/output encoding has been cleaned up significantly, stop hiding
    "Wide character in..." warnings, as they may conceal true problems.

diff --git a/lib/RT.pm b/lib/RT.pm
index 140d43e..67705c6 100644
--- a/lib/RT.pm
+++ b/lib/RT.pm
@@ -381,19 +381,9 @@ sub InitSignalHandlers {
 ## mechanism (see above).
 
     $SIG{__WARN__} = sub {
-        # The 'wide character' warnings has to be silenced for now, at least
-        # until HTML::Mason offers a sane way to process both raw output and
-        # unicode strings.
         # use 'goto &foo' syntax to hide ANON sub from stack
-        if( index($_[0], 'Wide character in ') != 0 ) {
-            unshift @_, $RT::Logger, qw(level warning message);
-            goto &Log::Dispatch::log;
-        }
-        # Return value is used only by RT::Test to filter warnings from
-        # reaching the Test::NoWarnings catcher.  If Log::Dispatch::log() ever
-        # starts returning 'IGNORE', we'll need to switch to something more
-        # clever.  I don't expect that to happen.
-        return 'IGNORE';
+        unshift @_, $RT::Logger, qw(level warning message);
+        goto &Log::Dispatch::log;
     };
 
 #When we call die, trap it and log->crit with the value of the die.
diff --git a/lib/RT/Test.pm b/lib/RT/Test.pm
index e956160..104e93a 100644
--- a/lib/RT/Test.pm
+++ b/lib/RT/Test.pm
@@ -648,12 +648,7 @@ sub __init_logging {
         $filter = $SIG{__WARN__};
     }
     $SIG{__WARN__} = sub {
-        if ($filter) {
-            my $status = $filter->(@_);
-            if ($status and $status eq 'IGNORE') {
-                return; # pretend the bad dream never happened
-            }
-        }
+        $filter->(@_) if $filter;
         # Avoid reporting this anonymous call frame as the source of the warning.
         goto &$Test_NoWarnings_Catcher;
     };

-----------------------------------------------------------------------


More information about the rt-commit mailing list