[Rt-commit] rt branch 5.0/shred-attachments-fts-index created. rt-5.0.4-20-gbedcf8bfd9
BPS Git Server
git at git.bestpractical.com
Sun May 28 10:43:57 UTC 2023
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "rt".
The branch, 5.0/shred-attachments-fts-index has been created
at bedcf8bfd9f534d32c70ad7d2c897b1ce077ff34 (commit)
- Log -----------------------------------------------------------------
commit bedcf8bfd9f534d32c70ad7d2c897b1ce077ff34
Author: Ruslan Zakirov <ruz at bestpractical.com>
Date: Wed May 24 08:39:52 2023 +0300
Wipeout full text index records during shredding
* Shredder::RawRecord allows us to delete a record in a table without
RT::Record mapping, for example AttachmentsIndex
* wipeout AttachmentsIndex if it's enabled
* write a test that works with mysql and Pg
* Remove some old outdated wording in shredder's docs
diff --git a/lib/RT/Attachment.pm b/lib/RT/Attachment.pm
index 2c50447a91..08466e54d4 100644
--- a/lib/RT/Attachment.pm
+++ b/lib/RT/Attachment.pm
@@ -1345,6 +1345,19 @@ sub __DependsOn {
);
push( @$list, $objs );
+ # fulltext index
+ my $fts = RT->Config->Get('FullTextSearch');
+ if ( $fts && $fts->{Indexed} && $fts->{Table} ) {
+ require RT::Shredder::RawRecord;
+ push @$list, RT::Shredder::RawRecord->new(
+ CurrentUser => $self->CurrentUser,
+ Table => $fts->{Table},
+ Columns => {
+ id => $self->Id,
+ },
+ );
+ }
+
$deps->_PushDependencies(
BaseObject => $self,
Flags => RT::Shredder::Constants::DEPENDS_ON,
diff --git a/lib/RT/Shredder.pm b/lib/RT/Shredder.pm
index 6baaf42e30..362c8b7058 100644
--- a/lib/RT/Shredder.pm
+++ b/lib/RT/Shredder.pm
@@ -351,6 +351,8 @@ sub CastObjectsToRecords
while( my $tmp = $targets->Next ) { push @res, $tmp };
} elsif ( UNIVERSAL::isa( $targets, 'RT::Record' ) ) {
push @res, $targets;
+ } elsif ( UNIVERSAL::isa( $targets, 'RT::Shredder::RawRecord' ) ) {
+ push @res, $targets;
} elsif ( UNIVERSAL::isa( $targets, 'ARRAY' ) ) {
foreach( @$targets ) {
push @res, $self->CastObjectsToRecords( Objects => $_ );
@@ -439,7 +441,7 @@ sub PutObject
my %args = ( Object => undef, @_ );
my $obj = $args{'Object'};
- unless( UNIVERSAL::isa( $obj, 'RT::Record' ) ) {
+ if( !UNIVERSAL::isa( $obj, 'RT::Record' ) && !UNIVERSAL::isa( $obj, 'RT::Shredder::RawRecord' ) ) {
RT::Shredder::Exception->throw( "Unsupported type '". (ref $obj || $obj || '(undef)')."'" );
}
diff --git a/lib/RT/Shredder/Plugin/SQLDump.pm b/lib/RT/Shredder/Plugin/SQLDump.pm
index c0366d5572..1e3f1f4644 100644
--- a/lib/RT/Shredder/Plugin/SQLDump.pm
+++ b/lib/RT/Shredder/Plugin/SQLDump.pm
@@ -87,6 +87,8 @@ sub Run
my %args = ( Object => undef, @_ );
my $query = $args{'Object'}->_AsInsertQuery;
+ return 1 unless $query;
+
$query .= "\n" unless $query =~ /\n$/;
utf8::encode($query) if utf8::is_utf8($query);
diff --git a/lib/RT/Shredder/RawRecord.pm b/lib/RT/Shredder/RawRecord.pm
new file mode 100644
index 0000000000..621ec81504
--- /dev/null
+++ b/lib/RT/Shredder/RawRecord.pm
@@ -0,0 +1,151 @@
+# BEGIN BPS TAGGED BLOCK {{{
+#
+# COPYRIGHT:
+#
+# This software is Copyright (c) 1996-2023 Best Practical Solutions, LLC
+# <sales at bestpractical.com>
+#
+# (Except where explicitly superseded by other copyright notices)
+#
+#
+# LICENSE:
+#
+# This work is made available to you under the terms of Version 2 of
+# the GNU General Public License. A copy of that license should have
+# been provided with this software, but in any event can be snarfed
+# from www.gnu.org.
+#
+# This work is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301 or visit their web page on the internet at
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.html.
+#
+#
+# CONTRIBUTION SUBMISSION POLICY:
+#
+# (The following paragraph is not intended to limit the rights granted
+# to you to modify and distribute this software under the terms of
+# the GNU General Public License and is only of importance to you if
+# you choose to contribute your changes and enhancements to the
+# community by submitting them to Best Practical Solutions, LLC.)
+#
+# By intentionally submitting any modifications, corrections or
+# derivatives to this work, or any other work intended for use with
+# Request Tracker, to Best Practical Solutions, LLC, you confirm that
+# you are the copyright holder for those contributions and you grant
+# Best Practical Solutions, LLC a nonexclusive, worldwide, irrevocable,
+# royalty-free, perpetual, license to use, copy, create derivative
+# works based on those contributions, and sublicense and distribute
+# those contributions and any derivatives thereof.
+#
+# END BPS TAGGED BLOCK }}}
+
+package RT::Shredder::RawRecord;
+
+use strict;
+use warnings FATAL => 'all';
+
+sub new {
+ my $proto = shift;
+ my $self = bless( {}, ref $proto || $proto );
+ $self->Set(@_);
+ return $self;
+}
+
+sub Set {
+ my $self = shift;
+ my %args = (@_);
+ my @keys = qw(Table Columns);
+ @$self{@keys} = @args{@keys};
+
+ $self->Load;
+
+ return;
+}
+
+sub UID {
+ my $self = shift;
+ return $self->{'UID'} if $self->{'UID'};
+
+ my $cols = map { "$_:" . $self->{'Columns'}{$_} }
+ sort keys %{ $self->{'Columns'} };
+ my $uid = join '-', ref $self, $RT::Organization, $self->Table, $cols;
+ return $self->{'UID'} = $uid;
+}
+
+sub Load {
+ my $self = shift;
+
+ my @cols = keys %{ $self->{'Columns'} };
+
+ my $dbh = $RT::Handle->dbh;
+ my $res = $dbh->selectall_arrayref(
+ "SELECT * FROM "
+ . $self->Table
+ . " WHERE "
+ . join( " AND ", map $dbh->quote_identifier($_) . " = ?", @cols ),
+ { Slice => {} },
+ @{ $self->{'Columns'} }{@cols},
+ );
+ unless ($res) {
+ die "Failed to load " . $self->UID . ": " . $dbh->errstr;
+ }
+
+ $self->{'records'} = $res;
+}
+
+sub _AsInsertQuery {
+ my $self = shift;
+ return "" unless $self->{'records'} && scalar @{ $self->{'records'} };
+
+ my $dbh = $RT::Handle->dbh;
+ my @cols = keys %{ $self->{'records'}[0] };
+
+ my $res = "INSERT INTO " . $self->Table;
+ $res .= "(" . join( ", ", map $dbh->quote_identifier($_), @cols ) . ")";
+ $res .= " VALUES\n";
+ for my $rec ( @{ $self->{'records'} } ) {
+ $res .= "\t("
+ . join( ", ", map { $dbh->quote( $rec->{$_} ) } @cols ) . "),\n";
+ }
+ $res =~ s/,\n$/;\n/;
+
+ return $res;
+}
+
+sub BeforeWipeout {
+ return 1;
+}
+
+sub Dependencies {
+ return RT::Shredder::Dependencies->new();
+}
+
+sub __Wipeout {
+ my $self = shift;
+ my $msg = $self->UID . " wiped out";
+
+ my $dbh = $RT::Handle->dbh;
+ my $query = "DELETE FROM " . $self->Table . " WHERE " . join(
+ " AND ",
+ map {
+ $dbh->quote_identifier($_) . "="
+ . $dbh->quote( $self->{'Columns'}{$_} )
+ } keys %{ $self->{'Columns'} }
+ )
+ . ";";
+
+ $dbh->do($query);
+
+ $RT::Logger->info($msg);
+}
+
+sub Table { return $_[0]->{'Table'} }
+
+1;
diff --git a/lib/RT/Shredder/Plugin/SQLDump.pm b/lib/RT/Test/FTS.pm
similarity index 62%
copy from lib/RT/Shredder/Plugin/SQLDump.pm
copy to lib/RT/Test/FTS.pm
index c0366d5572..5304de6499 100644
--- a/lib/RT/Shredder/Plugin/SQLDump.pm
+++ b/lib/RT/Test/FTS.pm
@@ -46,53 +46,60 @@
#
# END BPS TAGGED BLOCK }}}
-package RT::Shredder::Plugin::SQLDump;
-
use strict;
use warnings;
-use base qw(RT::Shredder::Plugin::Base::Dump);
-use RT::Shredder;
+package RT::Test::FTS;
-sub AppliesToStates { return 'after wiping dependencies' }
+require Test::More;
+require RT::Test;
-sub SupportArgs
-{
- my $self = shift;
- return $self->SUPER::SupportArgs, qw(file_name from_storage);
-}
+=head1 DESCRIPTION
+
+RT::Test::FTS - test suite utilities for testing with Full Text Search enabled
+
+=head1 FUNCTIONS
+
+=head2 setup_indexing
+
+ RT::Test::FTS->setup_indexing;
+
+Runs rt-setup-fulltext-index in silent mode with defaults.
+
+=cut
+
+sub setup_indexing {
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
-sub TestArgs
-{
my $self = shift;
- my %args = @_;
- $args{'from_storage'} = 1 unless defined $args{'from_storage'};
- my $file = $args{'file_name'} = RT::Shredder->GetFileName(
- FileName => $args{'file_name'},
- FromStorage => delete $args{'from_storage'},
+ my %args = (
+ 'no-ask' => 1,
+ command => $RT::SbinPath . '/rt-setup-fulltext-index',
+ dba => $ENV{'RT_DBA_USER'},
+ 'dba-password' => $ENV{'RT_DBA_PASSWORD'},
);
- open $args{'file_handle'}, ">:raw", $file
- or return (0, "Couldn't open '$file' for write: $!");
-
- return $self->SUPER::TestArgs( %args );
+ my ( $exit_code, $output ) = RT::Test->run_and_capture(%args);
+ Test::More::ok( !$exit_code, "setted up index" )
+ or Test::More::diag("output: $output");
}
-sub FileName { return $_[0]->{'opt'}{'file_name'} }
-sub FileHandle { return $_[0]->{'opt'}{'file_handle'} }
+=head2 sync_index
-sub Run
-{
- my $self = shift;
- return (0, 'no handle') unless my $fh = $self->{'opt'}{'file_handle'};
+ RT::Test::FTS->sync_index;
+
+Runs rt-fulltext-indexer to update index, run after creating attachments
+before executing searches.
- my %args = ( Object => undef, @_ );
- my $query = $args{'Object'}->_AsInsertQuery;
- $query .= "\n" unless $query =~ /\n$/;
+=cut
- utf8::encode($query) if utf8::is_utf8($query);
+sub sync_index {
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
- return 1 if print $fh $query;
- return (0, "Couldn't write to filehandle");
+ my $self = shift;
+ my %args = ( command => $RT::SbinPath . '/rt-fulltext-indexer', );
+ my ( $exit_code, $output ) = RT::Test->run_and_capture(%args);
+ Test::More::ok( !$exit_code, "setted up index" )
+ or Test::More::diag("output: $output");
}
1;
diff --git a/lib/RT/Test/Shredder.pm b/lib/RT/Test/Shredder.pm
index 4c18a54d70..6360ffc8b4 100644
--- a/lib/RT/Test/Shredder.pm
+++ b/lib/RT/Test/Shredder.pm
@@ -59,46 +59,30 @@ require Cwd;
RT::Shredder test suite utilities
-=head1 TESTING
-
-Since RT:Shredder 0.01_03 we have a test suite. You
-can run tests and see if everything works as expected
-before you try shredder on your actual data.
-Tests also help in the development process.
-
-The test suite uses SQLite databases to store data in individual files,
-so you could sun tests on your production servers without risking
-damage to your production data.
-
-You'll want to run the test suite almost every time you install or update
-the shredder distribution, especialy if you have local customizations of
-the DB schema and/or RT code.
-
-Tests are one thing you can write even if you don't know much perl,
-but want to learn more about RT's internals. New tests are very welcome.
-
=head2 WRITING TESTS
The shredder distribution has several files to help write new tests.
- t/shredder/utils.pl - this file, utilities
- t/00skeleton.t - skeleteton .t file for new tests
+ lib/RT/Test/Shredder.pm - this file, utilities
+ t/shredder/00skeleton.t - skeleteton .t file for new tests
All tests follow this algorithm:
- require "t/shredder/utils.pl"; # plug in utilities
- init_db(); # create new tmp RT DB and init RT API
+ use RT::Test::Shredder tests => undef; # plug in utilities
+ my $test = "RT::Test::Shredder"; # alias for RT::Test::Shredder
# create RT data you want to be always in the RT DB
# ...
- create_savepoint('mysp'); # create DB savepoint
+ $test->create_savepoint('clean'); # create DB savepoint
# create data you want delete with shredder
# ...
# run shredder on the objects you've created
# ...
# check that shredder deletes things you want
# this command will compare savepoint DB with current
- cmp_deeply( dump_current_and_savepoint('mysp'), "current DB equal to savepoint");
+ cmp_deeply( $test->dump_current_and_savepoint('mysp'), "current DB equal to savepoint");
# then you can create another object and delete it, then check again
+ # ...
+ done_testing();
Savepoints are named and you can create two or more savepoints.
diff --git a/t/fts/indexed_mysql.t b/t/fts/indexed_mysql.t
index 672b220723..658cff125c 100644
--- a/t/fts/indexed_mysql.t
+++ b/t/fts/indexed_mysql.t
@@ -5,33 +5,16 @@ use warnings;
use RT::Test tests => undef;
plan skip_all => 'Not mysql' unless RT->Config->Get('DatabaseType') eq 'mysql';
+use RT::Test::FTS;
+
RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Table => 'AttachmentsIndex' );
-setup_indexing();
+RT::Test::FTS->setup_indexing();
my $q = RT::Test->load_or_create_queue( Name => 'General' );
ok $q && $q->id, 'loaded or created queue';
my $queue = $q->Name;
-sub setup_indexing {
- my %args = (
- 'no-ask' => 1,
- command => $RT::SbinPath .'/rt-setup-fulltext-index',
- dba => $ENV{'RT_DBA_USER'},
- 'dba-password' => $ENV{'RT_DBA_PASSWORD'},
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "setted up index") or diag "output: $output";
-}
-
-sub sync_index {
- my %args = (
- command => $RT::SbinPath .'/rt-fulltext-indexer',
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "setted up index") or diag "output: $output";
-}
-
sub run_tests {
my @test = @_;
while ( my ($query, $checks) = splice @test, 0, 2 ) {
@@ -71,7 +54,7 @@ sub run_test {
{ Subject => 'third', Content => 'spanish' },
{ Subject => 'fourth', Content => 'german' },
);
-sync_index();
+RT::Test::FTS->sync_index();
run_tests(
"Content LIKE 'english'" => { first => 1, second => 0, third => 0, fourth => 0 },
diff --git a/t/fts/indexed_oracle.t b/t/fts/indexed_oracle.t
index a5b15bd825..bd634eb400 100644
--- a/t/fts/indexed_oracle.t
+++ b/t/fts/indexed_oracle.t
@@ -8,31 +8,12 @@ plan tests => 13;
RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, IndexName => 'rt_fts_index' );
-setup_indexing();
+RT::Test::FTS->setup_indexing();
my $q = RT::Test->load_or_create_queue( Name => 'General' );
ok $q && $q->id, 'loaded or created queue';
my $queue = $q->Name;
-sub setup_indexing {
- my %args = (
- 'no-ask' => 1,
- command => $RT::SbinPath .'/rt-setup-fulltext-index',
- dba => $ENV{'RT_DBA_USER'},
- 'dba-password' => $ENV{'RT_DBA_PASSWORD'},
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "setted up index") or diag "output: $output";
-}
-
-sub sync_index {
- my %args = (
- command => $RT::SbinPath .'/rt-fulltext-indexer',
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "synced the index") or diag "output: $output";
-}
-
sub run_tests {
my @test = @_;
while ( my ($query, $checks) = splice @test, 0, 2 ) {
@@ -70,7 +51,7 @@ sub run_test {
{ Subject => 'book', Content => 'book' },
{ Subject => 'bar', Content => 'bar' },
);
-sync_index();
+RT::Test::FTS->sync_index();
run_tests(
"Content LIKE 'book'" => { book => 1, bar => 0 },
diff --git a/t/fts/indexed_pg.t b/t/fts/indexed_pg.t
index 1494fded25..4ec2fd40b2 100644
--- a/t/fts/indexed_pg.t
+++ b/t/fts/indexed_pg.t
@@ -5,37 +5,20 @@ use warnings;
use RT::Test tests => undef;
plan skip_all => 'Not Pg' unless RT->Config->Get('DatabaseType') eq 'Pg';
+use RT::Test::FTS;
+
my ($major, $minor) = $RT::Handle->dbh->get_info(18) =~ /^0*(\d+)\.0*(\d+)/;
plan skip_all => "Need Pg 8.2 or higher; we have $major.$minor"
if "$major.$minor" < 8.2;
RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Column => 'ContentIndex', Table => 'AttachmentsIndex' );
-setup_indexing();
+RT::Test::FTS->setup_indexing();
my $q = RT::Test->load_or_create_queue( Name => 'General' );
ok $q && $q->id, 'loaded or created queue';
my $queue = $q->Name;
-sub setup_indexing {
- my %args = (
- 'no-ask' => 1,
- command => $RT::SbinPath .'/rt-setup-fulltext-index',
- dba => $ENV{'RT_DBA_USER'},
- 'dba-password' => $ENV{'RT_DBA_PASSWORD'},
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "setted up index") or diag "output: $output";
-}
-
-sub sync_index {
- my %args = (
- command => $RT::SbinPath .'/rt-fulltext-indexer',
- );
- my ($exit_code, $output) = RT::Test->run_and_capture( %args );
- ok(!$exit_code, "setted up index") or diag "output: $output";
-}
-
sub run_tests {
my @test = @_;
while ( my ($query, $checks) = splice @test, 0, 2 ) {
@@ -74,7 +57,7 @@ my $blase = Encode::decode_utf8("blasé");
{ Subject => 'fts test 1', Content => "book $blase" },
{ Subject => 'fts test 2', Content => "bars blasé", ContentType => 'text/html' },
);
-sync_index();
+RT::Test::FTS->sync_index();
my $book = $tickets[0];
my $bars = $tickets[1];
diff --git a/t/shredder/04fts.t b/t/shredder/04fts.t
new file mode 100644
index 0000000000..8cc3fc92c2
--- /dev/null
+++ b/t/shredder/04fts.t
@@ -0,0 +1,82 @@
+
+use strict;
+use warnings;
+
+use RT::Test tests => undef;
+plan skip_all => 'Only on Pg and mysql' unless RT->Config->Get('DatabaseType') =~ /mysql|Pg/;
+
+use RT::Test::FTS;
+RT->Config->Set( FullTextSearch => Enable => 1, Indexed => 1, Column => 'ContentIndex', Table => 'AttachmentsIndex' );
+RT::Test::FTS->setup_indexing();
+
+my $q = RT::Test->load_or_create_queue( Name => 'General' );
+ok $q && $q->id, 'loaded or created queue';
+
+# don't want to import as we are not on SQLite, but we want to use couple of utils
+require RT::Test::Shredder;
+
+{
+ my @tickets = RT::Test->create_tickets(
+ { Queue => $q->id },
+ { Subject => 'first', Content => 'english' },
+ );
+ $_->ApplyTransactionBatch for @tickets;
+
+ RT::Test::FTS->sync_index;
+
+ my @index_ids = index_ids(@tickets);
+ is scalar @index_ids, 1, 'one attachment indexed';
+
+ my $shredder = RT::Test::Shredder->shredder_new();
+ $shredder->PutObjects( Objects => \@tickets );
+ $shredder->WipeoutAll;
+
+ my $count = count_indexes(@index_ids);
+ is $count, 0, 'no attachment indexed';
+
+ RT::Test::Shredder->db_is_valid;
+
+ like get_dump($shredder), qr/AttachmentsIndex/, 'dump contains AttachmentsIndex';
+}
+
+# select directly from FTS table and get ids of indexed attachments
+sub index_ids {
+ my @tickets = @_;
+ my @ids = map { $_->id } @tickets;
+
+ my $dbh = $RT::Handle->dbh;
+ my $res = $dbh->selectcol_arrayref(
+ "SELECT a.id FROM AttachmentsIndex ai
+ JOIN Attachments a ON a.id = ai.id
+ JOIN Transactions txn ON a.TransactionId = txn.id AND txn.ObjectType = ?
+ WHERE txn.ObjectId IN (" . join( ',', ('?') x @ids ) . ")",
+ undef, 'RT::Ticket', @ids,
+ );
+ return @$res;
+}
+
+sub count_indexes {
+ my @ids = @_;
+ my $dbh = $RT::Handle->dbh;
+ my ($res) = $dbh->selectrow_array(
+ "SELECT COUNT(*) FROM AttachmentsIndex ai WHERE ai.id IN (" . join( ',', ('?') x @ids ) . ")",
+ undef, @ids,
+ );
+ return $res;
+}
+
+sub slurp {
+ my $fname = shift;
+
+ open my $fh, '<', $fname or die "Can't open $fname: $!";
+ return do { local $/; <$fh> };
+}
+
+sub get_dump {
+ my $shredder = shift;
+ my $dplugin = $shredder->{'dump_plugins'}[0];
+ my $fname = $dplugin->FileName;
+ return slurp($fname);
+}
+
+done_testing();
\ No newline at end of file
-----------------------------------------------------------------------
hooks/post-receive
--
rt
More information about the rt-commit
mailing list