[Rt-commit] rt branch, 4.6/mysql-utf8mb4, repushed
Michel Rodriguez
michel at bestpractical.com
Thu Jan 23 09:42:34 EST 2020
The branch 4.6/mysql-utf8mb4 was deleted and repushed:
was cf02e1942061458b61a7bb49a4e072b5fd25ee33
now 8cca778fee787fbcfa884822c6aff5e48fae7dcd
1: b6ab972188 ! 1: 09feb791d0 Support for utf8mb4 in mysql.
@@ -3,7 +3,7 @@
Support for utf8mb4 in mysql.
This allows 4-byte characters, such as emojis, in fields.
- This feature is supported in mysql 5.5.3 and later.
+ This feature is supported properly in mysql 5.7.7 and later.
It requires the database to be converted to utf8mb4 to
support those characters.
@@ -16,10 +16,12 @@
o A supported SQL database
- Currently supported: MySQL 5.1 - 5.7 with InnoDB support
-+ Currently supported: MySQL 5.5.3 - 5.7 with InnoDB support
- MariaDB 10.0 - 10.1 with InnoDB support
+- MariaDB 10.0 - 10.1 with InnoDB support
++ Currently supported: MySQL 5.7 with InnoDB support
++ MariaDB 10.2 or later with InnoDB support
Postgres 9.0 or later
Oracle 11g or later
+ SQLite 3.0 or later; for testing only, no
diff --git a/etc/schema.mysql b/etc/schema.mysql
--- a/etc/schema.mysql
2: dd950130cf ! 2: 7e61acceae Check that the database charset is utf8mb4.
@@ -33,4 +33,13 @@
return (1)
}
+@@
+ delete $item->{'BasedOn'};
+ }
+
+- }
++ }
+
+ my ( $return, $msg ) = $new_entry->Create(%$item);
+ unless( $return ) {
3: f1b780f45c ! 3: bc347b3f1d Explain utf8mb4 character set options.
@@ -15,7 +15,7 @@
+
+Unless the DB is shared with other applications that do not use this
+character set, it should be configured to use it by default. This
-+should be done in the server, client and mysqldump sections of the
++should be done in the server, client and mysqldump sections of the
+mysql config files.
+
+This is done by adding the following lines to the MySQL configuration:
@@ -40,7 +40,7 @@
+backups to be silently corrupted.
+
+
-+If the MySQL DB is shared with other applications and the default
++If the MySQL DB is shared with other applications and the default
+character set cannot be set to utf8mb4, the command to backup the
+database must set it explicitely:
+
@@ -49,7 +49,7 @@
+ | gzip > rt-`date +%Y%m%d`.sql.gz
+
+Restoring a backup is done the usual way, since the character set for
-+all tables is set to utf8mb4, there is no further need to tell MySQL
++all tables is set to utf8mb4, there is no further need to tell MySQL
+about it:
+
+ gunzip -c rt-20191125.sql.gz | mysql -uroot -p rt4
4: 2079d9171d = 4: b0837b7ddc Normalize character set/collation declaration for all tables.
5: e9a2e77979 ! 5: 052c2b8d93 Test for 4-byte utf-8 character searches.
@@ -29,15 +29,14 @@
+
+# setup the queue
+
-+my $q = RT::Queue->new(RT->SystemUser);
-+my $queue = 'queue'; # used globaly
-+$q->Create(Name => $queue);
-+ok ($q->id, "Created the queue");
++my $queue = 'General'; # used globaly
++my $q = RT::Test->load_or_create_queue( Name => $queue );
++ok( $q->id, "Loaded the queue" );
+
+my %cf = ( desc => create_cf( $q, 'desc'), char => create_cf( $q, 'char'), cat => create_cf( $q, 'cat') );
+my %cats = ( regular => 0, emoji => 0);
+
-+while( my( $desc, $char)= each %tickets) {
++while( my( $desc, $char)= each %tickets ) {
+
+ my $cat = ord( $char ) > 100000 ? 'emoji' : 'regular';
+ $cats{$cat}++;
@@ -57,21 +56,21 @@
+
+my $tix = RT::Tickets->new(RT->SystemUser);
+$tix->FromSQL("Queue = '$queue'");
-+is($tix->Count, scalar( keys %tickets), "found all the tickets")
++is( $tix->Count, scalar( keys %tickets ), "found all the tickets" )
+ or diag "wrong results from SQL:\n". $tix->BuildSelectCountQuery;
+
-+while( my( $cat, $nb) = each %cats) {
++while( my( $cat, $nb ) = each %cats ) {
+ my $tix_cat = RT::Tickets->new(RT->SystemUser);
+ $tix_cat->FromSQL("Queue = '$queue' AND $cf{cat}->{search} = '$cat'");
-+ is($tix_cat->Count, $nb, "found $nb $cat ticket (from cat CF)")
++ is( $tix_cat->Count, $nb, "found $nb $cat ticket (from cat CF)" )
+ or diag "wrong results from SQL:\n". $tix_cat->BuildSelectCountQuery;
+}
+
-+while( my( $desc, $char)= each %tickets) {
++while( my( $desc, $char ) = each %tickets ) {
+ my $subject = subject( $desc, $char );
-+ test_search( $desc, $char, 'Subject', $char);
-+ test_search( $desc, $char, $cf{desc}->{search}, $desc);
-+ test_search( $desc, $char, $cf{char}->{search}, $char);
++ test_search( $desc, $char, 'Subject', $char );
++ test_search( $desc, $char, $cf{desc}->{search}, $desc );
++ test_search( $desc, $char, $cf{char}->{search}, $char );
+}
+
+done_testing;
@@ -83,9 +82,8 @@
+
+sub create_cf {
+ my( $q, $name)= @_;
-+ my $cf = RT::CustomField->new(RT->SystemUser);
-+ $cf->Create(Name => $name, Type => 'Freeform', MaxValues => 0, Queue => $q->id);
-+ ok($cf->id, "Created the $name CF");
++ my $cf = RT::Test->load_or_create_custom_field( Name => $name, Type => 'Freeform', MaxValues => 0, Queue => $q->id );
++ ok( $cf->id, "Created the $name CF" );
+ my $create = "CustomField-" . $cf->id; # key used to create the CF
+ my $search = "CF.$name"; # field in search
+ return { name => $name, create => $create, search => $search };
@@ -97,14 +95,11 @@
+ my $tix = RT::Tickets->new(RT->SystemUser);
+ $tix->FromSQL("Queue = '$queue' AND $field LIKE '$to_search'");
+
-+ is($tix->Count, 1, "found $to_search ticket (from $field)")
++ is( $tix->Count, 1, "found $to_search ticket (from $field)" )
+ or diag "wrong results from SQL:\n". $tix->BuildSelectCountQuery;
+
+ my $ticket = $tix->First;
+ return if ! $ticket;
-+ my $got = $ticket->Subject;
-+ my $expected = subject( $desc, $char);
-+ is( $got, $expected, "$char, $field: Subject is right");
++ is( $ticket->Subject, subject( $desc, $char), "$char, $field: Subject is right" );
+}
-+
6: 7d57f69715 = 6: 984b3a7889 Change the full text indexer charset to utf8mb4.
7: 913b006f98 ! 7: 8bd009b4cf Alter all relevant tables to use the utf8mb4 charset.
@@ -17,18 +17,25 @@
+ ALTER TABLE `Transactions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `Scrips` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `ObjectScrips` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
++ ALTER TABLE `ACL` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
++ ALTER TABLE `GroupMembers` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
++ ALTER TABLE `CachedGroupMembers` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `Users` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `Tickets` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `ScripActions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `Templates` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `ObjectCustomFieldValues` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `CustomFields` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
++ ALTER TABLE `ObjectCustomFields` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `CustomFieldValues` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `Attributes` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `Classes` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `Articles` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `Topics` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
++ ALTER TABLE `ObjectTopics` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
++ ALTER TABLE `ObjectClasses` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `Assets` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `Catalogs` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ ALTER TABLE `CustomRoles` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
++ ALTER TABLE `ObjectCustomRoles` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
8: cf02e19420 ! 8: ac4f8b2a03 Improve mysql/mariadb version checks.
@@ -2,9 +2,21 @@
Improve mysql/mariadb version checks.
- If the version is > 9.9 then the DB is assumend to be MariaDB
- MySQL version should be >= 5.7
- MariaDB version should be >= 10.2
+ MySQL version should be >= 5.7.7
+ MariaDB version should be >= 10.2.5
+
+ https://dev.mysql.com/doc/relnotes/mysql/5.7/en/news-5-7-7.html#mysqld-5-7-7-feature
+ https://mariadb.com/kb/en/changes-improvements-in-mariadb-102/
+ https://mariadb.com/kb/en/mariadb-1025-changelog/ (search for utf8)
+
+ Before this version utf8mb4 tables could not have indexes with type VARCHAR(255):
+ the default size for index entries was 767 bytes, which is enough for 255 chars
+ stored as at most 3 chars (the utf8 format), but not as 4 bytes (utf8mb4).
+ 5.7.7 sets the default index size to 3072, so all of RT indexes are now OK.
+
+ Detection of MariaDB is done on the version string (it must match "mariadb"),
+ which is the case in both Debian and RedHat based distributions, and likely
+ everywhere else.
diff --git a/lib/RT/Handle.pm b/lib/RT/Handle.pm
--- a/lib/RT/Handle.pm
@@ -21,7 +33,7 @@
- elsif( cmp_version( $version, '4.1') >= 0 ) {
- $self->dbh->do("SET NAMES 'utf8'");
- }
-+ # set the character set
++ # set the character set
+ $self->dbh->do("SET NAMES 'utf8mb4'");
}
elsif ( $db_type eq 'Pg' ) {
@@ -33,11 +45,12 @@
- my $min_version = '5.6';
- return (0, "RT is unsupported on MySQL versions before $min_version. Your version is $version.")
- if cmp_version( $version, $min_version) < 0;
-+ # MySQL and MariaDB are both 'mysql' type. mariadb 10.2 is ~ mysql 5.7
-+ my $mysql_max_version = '9.9'; # will have to be fixed when MySQL goes past this
-+ my $is_mariadb = cmp_version( $version, $mysql_max_version) < 0 ? 0 : 1;
-+ my $mysql_min_version = '5.7';
-+ my $mariadb_min_version = '10.2';
++ # MySQL and MariaDB are both 'mysql' type. mariadb 10.2 is ~ mysql 5.7
++ # the minimum version supported is MySQL 5.7.7 / MariaDB 10.2.2
++ # the version string for MariaDB includes "MariaDB" in the distributions I looked at (Debian/RedHat)
++ my $is_mariadb = $version =~ m{mariadb}i ? 0 : 1;
++ my $mysql_min_version = '5.7.7'; # so index sizes with utf8mb4 allow for VARCHAR(255) fields
++ my $mariadb_min_version = '10.2.5'; # that's when the relevant bit of MySQL 5.7 got merged
+
+ return (0, "RT is unsupported on MySQL versions before $mysql_min_version. Your version is $version.")
+ if ! $is_mariadb && cmp_version( $version, $mysql_min_version) < 0;
@@ -55,3 +68,4 @@
# - checking all columns would be expensive
my $allowed_charset = 'utf8mb4';
my $column_info = $dbh->selectrow_hashref( "show full columns from Tickets where Field = 'Subject'" );
+
-: ------- > 9: 8cca778fee Simplify checks on max_allowed_packet since MySQL version is now higher.
More information about the rt-commit
mailing list