[Rt-commit] rt branch, 4.4/mysql-replace-4-bytes-utf8-data, created. rt-4.4.3-63-ga0a64bece

[? sunnavy]

The branch, 4.4/mysql-replace-4-bytes-utf8-data has been created
        at  a0a64bece1a7e370e2c35c86a25beb32c4fc292b (commit)

- Log -----------------------------------------------------------------
commit 07d968d825f90d90f605b7d1ae42484c586be590
Author: sunnavy <sunnavy at bestpractical.com>
Date:   Wed Oct 24 04:36:06 2018 +0800

    Remove 4 bytes utf8 data for MySQL utf8 charset
    
    This could be dropped when we updated MySQL charset to "utf8mb4".
    
    Note that we need to protect instances that already migrated to "utf8mb4".

diff --git a/lib/RT/Handle.pm b/lib/RT/Handle.pm
index eaf3d76e3..91a54db76 100644
--- a/lib/RT/Handle.pm
+++ b/lib/RT/Handle.pm
@@ -130,6 +130,11 @@ sub Connect {
         my $version = $self->DatabaseVersion;
         ($version) = $version =~ /^(\d+\.\d+)/;
         $self->dbh->do("SET NAMES 'utf8'") if $version >= 4.1;
+
+        if ( $self->dbh->selectrow_arrayref("SHOW TABLES LIKE 'Tickets'") ) {
+            $self->{_unsafe_4bytes_utf8} = 1
+              unless $self->dbh->selectrow_arrayref("SHOW CREATE TABLE Tickets")->[1] =~ /charset=utf8mb4/i;
+        }
     }
     elsif ( $db_type eq 'Pg' ) {
         my $version = $self->DatabaseVersion;
diff --git a/lib/RT/Record.pm b/lib/RT/Record.pm
index 908cde342..4a025194e 100644
--- a/lib/RT/Record.pm
+++ b/lib/RT/Record.pm
@@ -295,6 +295,31 @@ sub Create {
         }
     }
 
+    if ( $RT::Handle->{_unsafe_4bytes_utf8} ) {
+
+        # This won't be necessary when DB schema is changed to utf8mb4.
+        for my $key ( keys %attribs ) {
+            if ( $attribs{$key} && $self->_Accessible( $key, 'type' ) =~ /text|varchar/ ) {
+                my $value;
+                if ( Encode::is_utf8( $attribs{$key} ) ) {
+                    $value = Encode::encode( 'UTF-8', $attribs{$key} );
+                }
+                else {
+                    $value = $attribs{$key};
+                }
+
+                if ( $value =~ s!(?:[\xF0-\xF7]...)+! !g ) {
+                    RT->Logger->debug("Removed 4 bytes UTF-8 characters from $attribs{$key}");
+                    if ( Encode::is_utf8( $attribs{$key} ) ) {
+                        $attribs{$key} = Encode::decode( 'UTF-8', $value );
+                    }
+                    else {
+                        $attribs{$key} = $value;
+                    }
+                }
+            }
+        }
+    }
 
 
     my ($sec,$min,$hour,$mday,$mon,$year,$wday,$ydaym,$isdst,$offset) = gmtime();

commit d4ef71f4e6271b2d82356d5b004bbd59d4704733
Author: sunnavy <sunnavy at bestpractical.com>
Date:   Thu Oct 25 03:44:26 2018 +0800

    Remove duplicated tests

diff --git a/t/web/ticket-create-utf8.t b/t/web/ticket-create-utf8.t
index ebb2d5eab..e1e28f3cc 100644
--- a/t/web/ticket-create-utf8.t
+++ b/t/web/ticket-create-utf8.t
@@ -2,7 +2,7 @@
 use strict;
 use warnings;
 
-use RT::Test tests => 43;
+use RT::Test tests => undef;
 
 my $ru_test = "\x{442}\x{435}\x{441}\x{442}";
 my $ru_support = "\x{43f}\x{43e}\x{434}\x{434}\x{435}\x{440}\x{436}\x{43a}\x{430}";
@@ -63,26 +63,4 @@ foreach my $test_str ( $ru_test, $l1_test ) {
     }
 }
 
-# create a ticket with a subject and content
-foreach my $test_str ( $ru_test, $l1_test ) {
-    foreach my $support_str ( $ru_support, $l1_support ) {
-        ok $m->goto_create_ticket( $q ), "go to create ticket";
-        $m->form_name('TicketCreate');
-        $m->field( Subject => $test_str );
-        $m->field( Content => $support_str );
-        $m->submit;
-
-        $m->content_like( 
-            qr{<td\s+class="message-header-value\s*"[^>]*>\s*\Q$test_str\E\s*</td>}i,
-            'header on the page'
-        );
-        $m->content_contains(
-            $support_str,
-            'content on the page'
-        );
-
-        my $ticket = RT::Test->last_ticket;
-        is $ticket->Subject, $test_str, "correct subject";
-    }
-}
-
+done_testing;

commit a0a64bece1a7e370e2c35c86a25beb32c4fc292b
Author: sunnavy <sunnavy at bestpractical.com>
Date:   Thu Oct 25 03:45:31 2018 +0800

    Test 4 bytes utf8 data

diff --git a/t/web/ticket-create-utf8.t b/t/web/ticket-create-utf8.t
index e1e28f3cc..ad00fb0e5 100644
--- a/t/web/ticket-create-utf8.t
+++ b/t/web/ticket-create-utf8.t
@@ -63,4 +63,10 @@ foreach my $test_str ( $ru_test, $l1_test ) {
     }
 }
 
+use utf8;
+ok $m->goto_create_ticket($q), "go to create ticket";
+$m->submit_form_ok( { form_name => 'TicketCreate', fields => { Subject => 'test😄emoji' } } );
+my $ticket = RT::Test->last_ticket;
+is $ticket->Subject, $RT::Handle->{_unsafe_4bytes_utf8} ? 'test emoji' : 'test😄emoji', "correct subject";
+
 done_testing;

-----------------------------------------------------------------------