[Rt-commit] rt branch, 4.4/speed-up-tests-by-making-database-non-durable, created. rt-4.4.4-128-g0becfaebf9

[Dianne Skoll]

The branch, 4.4/speed-up-tests-by-making-database-non-durable has been created
        at  0becfaebf93d6dd0462dfba577c0be2ca994d4d6 (commit)

- Log -----------------------------------------------------------------
commit 4f5fec8dbcc86511fff502f4db5763c499623b7a
Author: Dianne Skoll <dianne at bestpractical.com>
Date:   Tue Aug 4 14:38:00 2020 -0400

    Ticket #230935: Speed up tests by making database back-ends non-durable.
    
    Cherry-picked from 5.0/speed-up-tests-by-making-database-non-durable
    (commit 00b7d2e57c4fae3641e90b50d4c383361e841252)
    
    For SQLite, we use: "pragma synchronous = 0" to turn off most fsync() calls.
    This provides a huge improvement
    
    For PostgreSQL, we use: "set synchronous_commit = OFF" for a modest
    improvement.
    
    Nothing was implemented for MySQL as everything I tried made very little
    difference.
    
    For PostgreSQL, setting "fsync = off" in the configuration file
    provides a large improvement.  This change can only be done in
    the configuration file, unfortunately.
    
    For MariaDB, nothing I tried other than putting the database
    files in a ramdisk helped much.

diff --git a/lib/RT/Handle.pm b/lib/RT/Handle.pm
index 29e929e7b7..4e79421569 100644
--- a/lib/RT/Handle.pm
+++ b/lib/RT/Handle.pm
@@ -138,9 +138,19 @@ sub Connect {
         $self->dbh->do("SET bytea_output = 'escape'") if $version >= 9.0;
     }
 
+    PostConnectActions($db_type, $self->dbh);
     $self->dbh->{'LongReadLen'} = RT->Config->Get('MaxAttachmentSize');
 }
 
+=head2 PostConnectActions
+
+Perform any post-connect queries.  In production, this function
+does nothing; it exists to be overridden by test code.
+
+=cut
+sub PostConnectActions {
+}
+
 =head2 BuildDSN
 
 Build the DSN for the RT database. Doesn't take any parameters, draws all that
diff --git a/lib/RT/Test.pm b/lib/RT/Test.pm
index d7199d6b0b..fae85309bf 100644
--- a/lib/RT/Test.pm
+++ b/lib/RT/Test.pm
@@ -214,6 +214,30 @@ sub import {
     __PACKAGE__->export_to_level($level);
 }
 
+sub MakeDatabaseNonDurableForTesting {
+    my ($db_type, $dbh) = @_;
+
+    # If environment variable RT_TEST_DB_DURABLE is non-zero,
+    # don't make the DB non-durable.
+    return if $ENV{RT_TEST_DB_DURABLE};
+
+    if ($db_type =~ /sqlite/i) {
+        #print STDERR "Sending PRAGMA synchronous = 0;\n";
+        if (!$dbh->do(q{PRAGMA synchronous = 0;})) {
+            print STDERR "PRAGMA synchronous failed!\n";
+        }
+    } elsif ($db_type =~ /pg/i) {
+        #print STDERR "Sending SET synchronous_commit = OFF;\n";
+        if (!$dbh->do(q{SET synchronous_commit = OFF;})) {
+            print STDERR "SET synchronous_commit = OFF failed!\n";
+        }
+    }
+
+    # For MySQL, none of the settings mentioned at
+    # http://www.tocker.ca/2013/11/04/reducing-mysql-durability-for-testing.html
+    # helped much, so we don't bother doing anything. :(
+}
+
 sub is_empty($;$) {
     my ($v, $d) = shift;
     local $Test::Builder::Level = $Test::Builder::Level + 1;
@@ -622,6 +646,11 @@ sub _get_dbh {
         my $msg = "Failed to connect to $dsn as user '$user': ". $DBI::errstr;
         print STDERR $msg; exit -1;
     }
+
+    # Do what we can to make DB handle less durable to speed up tests
+    if ($dsn =~ /dbi:([^:]+):/i) {
+        MakeDatabaseNonDurableForTesting($1, $dbh);
+    }
     return $dbh;
 }
 
@@ -678,6 +707,17 @@ sub __reconnect_rt {
     # look at %DBIHandle and $PrevHandle in DBIx::SB::Handle for explanation
     $RT::Handle = RT::Handle->new;
     $RT::Handle->dbh( undef );
+
+    # Override RT::Handle::PostConnectActions
+    {
+        no warnings 'redefine';
+        *RT::Handle::PostConnectActions = sub {
+            my ($db_type, $dbh) = @_;
+            MakeDatabaseNonDurableForTesting($db_type, $dbh);
+        };
+        use warnings 'redefine';
+    }
+
     $RT::Handle->Connect(
         $as_dba
         ? (User => $ENV{RT_DBA_USER}, Password => $ENV{RT_DBA_PASSWORD})

commit 0becfaebf93d6dd0462dfba577c0be2ca994d4d6
Author: Dianne Skoll <dianne at bestpractical.com>
Date:   Wed Aug 5 08:56:27 2020 -0400

    Document (dangerous) DB optimizations to improve test speed.

diff --git a/docs/hacking.pod b/docs/hacking.pod
index 15742fb759..afed926041 100644
--- a/docs/hacking.pod
+++ b/docs/hacking.pod
@@ -214,6 +214,28 @@ running the tests in parallel.
 
     RT_TEST_PARALLEL=1 prove -l -j4 t/customfields/*.t
 
+=head2 Database Optimizations for Testing
+
+To improve the speed of testing, you may adjust the database configuration
+to optimize speed at the expense of losing durability (and potential database
+corruption.)
+
+For SQLite, the test suite automatically uses C<PRAGMA synchronous = 0> to
+eliminate many calls to C<fsync>, thereby greatly reducing disk I/O.
+
+For PostgreSQL, the test suite automatically uses C<SET
+synchronous_commit = OFF> to improve performance.  However, a much
+greater performance improvement can be realized by setting C<fsync =
+off> in the C<postgresql.conf> file.  (DO NOT set C<fsync = off> in a
+production system; doing so risks corrupting your entire PostgreSQL
+database.)
+
+For MySQL or MariaDB, the C<eatmydata> wrapper at L<https://www.flamingspork.com/projects/libeatmydata/>
+may be used to wrap the MySQL or MariaDB daemon; this uses C<LD_PRELOAD>
+to eliminate calls to C<fsync> and similar system calls.  (DO NOT user
+C<eatmydata> in a production system; doing so risks corrupting your entire
+MySQL database.)
+
 =head2 Tests with Docker
 
 RT has a Dockerfile that can be used to run the test suite for some configurations.

-----------------------------------------------------------------------