[Rt-commit] rt branch, 4.6/mysql-utf8mb4, repushed

Michel Rodriguez michel at bestpractical.com
Thu Jan 23 09:42:34 EST 2020


The branch 4.6/mysql-utf8mb4 was deleted and repushed:
       was cf02e1942061458b61a7bb49a4e072b5fd25ee33
       now 8cca778fee787fbcfa884822c6aff5e48fae7dcd

1: b6ab972188 ! 1: 09feb791d0 Support for utf8mb4 in mysql.
    @@ -3,7 +3,7 @@
         Support for utf8mb4 in mysql.
         
         This allows 4-byte characters, such as emojis, in fields.
    -    This feature is supported in mysql 5.5.3 and later.
    +    This feature is supported properly in mysql 5.7.7 and later.
         
         It requires the database to be converted to utf8mb4 to
         support those characters.
    @@ -16,10 +16,12 @@
      o   A supported SQL database
      
     -        Currently supported:  MySQL 5.1 - 5.7 with InnoDB support
    -+        Currently supported:  MySQL 5.5.3 - 5.7 with InnoDB support
    -                               MariaDB 10.0 - 10.1 with InnoDB support
    +-                              MariaDB 10.0 - 10.1 with InnoDB support
    ++        Currently supported:  MySQL 5.7 with InnoDB support
    ++                              MariaDB 10.2 or later with InnoDB support
                                    Postgres 9.0 or later
                                    Oracle 11g or later
    +                               SQLite 3.0 or later; for testing only, no
     
     diff --git a/etc/schema.mysql b/etc/schema.mysql
     --- a/etc/schema.mysql
2: dd950130cf ! 2: 7e61acceae Check that the database charset is utf8mb4.
    @@ -33,4 +33,13 @@
          return (1)
      }
      
    +@@
    +                     delete $item->{'BasedOn'};
    +                 }
    + 
    +-            } 
    ++            }
    + 
    +             my ( $return, $msg ) = $new_entry->Create(%$item);
    +             unless( $return ) {
     
3: f1b780f45c ! 3: bc347b3f1d Explain utf8mb4 character set options.
    @@ -15,7 +15,7 @@
     +
     +Unless the DB is shared with other applications that do not use this
     +character set, it should be configured to use it by default. This
    -+should be done in the server, client and mysqldump sections of the 
    ++should be done in the server, client and mysqldump sections of the
     +mysql config files.
     +
     +This is done by adding the following lines to the MySQL configuration:
    @@ -40,7 +40,7 @@
     +backups to be silently corrupted.
     +
     +
    -+If the MySQL DB is shared with other applications and the default 
    ++If the MySQL DB is shared with other applications and the default
     +character set cannot be set to utf8mb4, the command to backup the
     +database must set it explicitely:
     +
    @@ -49,7 +49,7 @@
     +        | gzip > rt-`date +%Y%m%d`.sql.gz
     +
     +Restoring a backup is done the usual way, since the character set for
    -+all tables is set to utf8mb4, there is no further need to tell MySQL 
    ++all tables is set to utf8mb4, there is no further need to tell MySQL
     +about it:
     +
     +    gunzip -c rt-20191125.sql.gz | mysql -uroot -p rt4
4: 2079d9171d = 4: b0837b7ddc Normalize character set/collation declaration for all tables.
5: e9a2e77979 ! 5: 052c2b8d93 Test for 4-byte utf-8 character searches.
    @@ -29,15 +29,14 @@
     +
     +# setup the queue
     +
    -+my $q = RT::Queue->new(RT->SystemUser);
    -+my $queue = 'queue';  # used globaly
    -+$q->Create(Name => $queue);
    -+ok ($q->id, "Created the queue");
    ++my $queue = 'General';  # used globaly
    ++my $q = RT::Test->load_or_create_queue( Name => $queue );
    ++ok( $q->id, "Loaded the queue" );
     +
     +my %cf = ( desc => create_cf( $q, 'desc'), char => create_cf( $q, 'char'), cat => create_cf( $q, 'cat') );
     +my %cats = ( regular => 0, emoji => 0);
     +
    -+while( my( $desc, $char)= each %tickets) {
    ++while( my( $desc, $char)= each %tickets ) {
     +
     +    my $cat = ord( $char ) > 100000 ? 'emoji' : 'regular';
     +    $cats{$cat}++;
    @@ -57,21 +56,21 @@
     +
     +my $tix = RT::Tickets->new(RT->SystemUser);
     +$tix->FromSQL("Queue = '$queue'");
    -+is($tix->Count, scalar( keys %tickets), "found all the tickets")
    ++is( $tix->Count, scalar( keys %tickets ), "found all the tickets" )
     +    or diag "wrong results from SQL:\n". $tix->BuildSelectCountQuery;
     +
    -+while( my( $cat, $nb) = each %cats) {
    ++while( my( $cat, $nb ) = each %cats ) {
     +    my $tix_cat = RT::Tickets->new(RT->SystemUser);
     +    $tix_cat->FromSQL("Queue = '$queue' AND $cf{cat}->{search} = '$cat'");
    -+    is($tix_cat->Count, $nb, "found $nb $cat ticket (from cat CF)")
    ++    is( $tix_cat->Count, $nb, "found $nb $cat ticket (from cat CF)" )
     +        or diag "wrong results from SQL:\n". $tix_cat->BuildSelectCountQuery;
     +}
     +
    -+while( my( $desc, $char)= each %tickets) {
    ++while( my( $desc, $char ) = each %tickets ) {
     +    my $subject = subject( $desc, $char );
    -+    test_search( $desc, $char, 'Subject', $char);
    -+    test_search( $desc, $char, $cf{desc}->{search}, $desc);
    -+    test_search( $desc, $char, $cf{char}->{search}, $char);
    ++    test_search( $desc, $char, 'Subject', $char );
    ++    test_search( $desc, $char, $cf{desc}->{search}, $desc );
    ++    test_search( $desc, $char, $cf{char}->{search}, $char );
     +}
     +
     +done_testing;
    @@ -83,9 +82,8 @@
     +
     +sub create_cf {
     +    my( $q, $name)= @_;
    -+    my $cf = RT::CustomField->new(RT->SystemUser);
    -+    $cf->Create(Name => $name, Type => 'Freeform', MaxValues => 0, Queue => $q->id);
    -+    ok($cf->id, "Created the $name CF");
    ++    my $cf = RT::Test->load_or_create_custom_field( Name => $name, Type => 'Freeform', MaxValues => 0, Queue => $q->id );
    ++    ok( $cf->id, "Created the $name CF" );
     +    my $create = "CustomField-" . $cf->id; # key used to create the CF
     +    my $search = "CF.$name";           # field in search
     +    return { name => $name, create => $create, search => $search };
    @@ -97,14 +95,11 @@
     +    my $tix = RT::Tickets->new(RT->SystemUser);
     +    $tix->FromSQL("Queue = '$queue' AND $field LIKE '$to_search'");
     +
    -+    is($tix->Count, 1, "found $to_search ticket (from $field)")
    ++    is( $tix->Count, 1, "found $to_search ticket (from $field)" )
     +        or diag "wrong results from SQL:\n". $tix->BuildSelectCountQuery;
     +
     +    my $ticket = $tix->First;
     +    return if ! $ticket;
    -+    my $got =  $ticket->Subject;
    -+    my $expected = subject( $desc, $char);
    -+    is( $got, $expected, "$char, $field: Subject is right");
    ++    is( $ticket->Subject, subject( $desc, $char), "$char, $field: Subject is right" );
     +}
    -+
     
6: 7d57f69715 = 6: 984b3a7889 Change the full text indexer charset to utf8mb4.
7: 913b006f98 ! 7: 8bd009b4cf Alter all relevant tables to use the utf8mb4 charset.
    @@ -17,18 +17,25 @@
     + ALTER TABLE `Transactions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `Scrips` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `ObjectScrips` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
    ++ ALTER TABLE `ACL` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
    ++ ALTER TABLE `GroupMembers` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
    ++ ALTER TABLE `CachedGroupMembers` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `Users` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `Tickets` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `ScripActions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `Templates` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `ObjectCustomFieldValues` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `CustomFields` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
    ++ ALTER TABLE `ObjectCustomFields` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `CustomFieldValues` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `Attributes` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `Classes` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `Articles` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `Topics` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
    ++ ALTER TABLE `ObjectTopics` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
    ++ ALTER TABLE `ObjectClasses` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `Assets` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `Catalogs` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     + ALTER TABLE `CustomRoles` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
    ++ ALTER TABLE `ObjectCustomRoles` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
     
8: cf02e19420 ! 8: ac4f8b2a03 Improve mysql/mariadb version checks.
    @@ -2,9 +2,21 @@
     
         Improve mysql/mariadb version checks.
         
    -    If the version is > 9.9 then the DB is assumend to be MariaDB
    -    MySQL version should be >= 5.7
    -    MariaDB version should be >= 10.2
    +    MySQL version should be >= 5.7.7
    +    MariaDB version should be >= 10.2.5
    +    
    +    https://dev.mysql.com/doc/relnotes/mysql/5.7/en/news-5-7-7.html#mysqld-5-7-7-feature
    +    https://mariadb.com/kb/en/changes-improvements-in-mariadb-102/
    +    https://mariadb.com/kb/en/mariadb-1025-changelog/ (search for utf8)
    +    
    +    Before this version utf8mb4 tables could not have indexes with type VARCHAR(255):
    +    the default size for index entries was 767 bytes, which is enough for 255 chars
    +    stored as at most 3 chars (the utf8 format), but not as 4 bytes (utf8mb4).
    +    5.7.7 sets the default index size to 3072, so all of RT indexes are now OK.
    +    
    +    Detection of MariaDB is done on the version string (it must match "mariadb"),
    +    which is the case in both Debian and RedHat based distributions, and likely
    +    everywhere else.
     
     diff --git a/lib/RT/Handle.pm b/lib/RT/Handle.pm
     --- a/lib/RT/Handle.pm
    @@ -21,7 +33,7 @@
     -        elsif( cmp_version( $version, '4.1') >= 0 ) {
     -            $self->dbh->do("SET NAMES 'utf8'");
     -        }
    -+        # set the character set 
    ++        # set the character set
     +        $self->dbh->do("SET NAMES 'utf8mb4'");
          }
          elsif ( $db_type eq 'Pg' ) {
    @@ -33,11 +45,12 @@
     -        my $min_version = '5.6';
     -        return (0, "RT is unsupported on MySQL versions before $min_version.  Your version is $version.")
     -            if cmp_version( $version, $min_version) < 0;
    -+        # MySQL and MariaDB are both 'mysql' type. mariadb 10.2 is ~ mysql 5.7  
    -+        my $mysql_max_version = '9.9';    # will have to be fixed when MySQL goes past this
    -+        my $is_mariadb = cmp_version( $version, $mysql_max_version) < 0 ? 0 : 1;
    -+        my $mysql_min_version = '5.7';
    -+        my $mariadb_min_version = '10.2';
    ++        # MySQL and MariaDB are both 'mysql' type. mariadb 10.2 is ~ mysql 5.7
    ++        # the minimum version supported is MySQL 5.7.7 / MariaDB 10.2.2
    ++        # the version string for MariaDB includes "MariaDB" in the distributions I looked at (Debian/RedHat)
    ++        my $is_mariadb = $version =~ m{mariadb}i  ? 0 : 1;
    ++        my $mysql_min_version = '5.7.7';    # so index sizes with utf8mb4 allow for VARCHAR(255) fields
    ++        my $mariadb_min_version = '10.2.5'; # that's when the relevant bit of MySQL 5.7 got merged
     +
     +        return (0, "RT is unsupported on MySQL versions before $mysql_min_version.  Your version is $version.")
     +            if ! $is_mariadb && cmp_version( $version, $mysql_min_version) < 0;
    @@ -55,3 +68,4 @@
              #  - checking all columns would be expensive
              my $allowed_charset = 'utf8mb4';
              my $column_info = $dbh->selectrow_hashref( "show full columns from Tickets where Field = 'Subject'" );
    +
-:  ------- > 9: 8cca778fee Simplify checks on max_allowed_packet since MySQL version is now higher.



More information about the rt-commit mailing list